From 89806bd9f72163e867446040a32104a8a5986c27 Mon Sep 17 00:00:00 2001 From: ahraz <38987266+ahrazzle@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:38:09 -0600 Subject: [PATCH 1/9] feat(mcproof): add v0.2 29-theorem library, label-free repair loop, and sandbox/statement guards --- README.md | 21 +- .../sealed_holdout_repair_loop_receipt.json | 22682 ++++++++++++++++ .../repair_loop/validation_repair_b0.json | 5788 ++++ .../repair_loop/validation_repair_p2b.json | 7020 +++++ .../validation_repair_summary.json | 268 + .../p2b/v01_plane1_snapshot/lean/Mcproof.lean | 34 + .../lean/Mcproof/Proofs.lean | 74 + .../lean/Mcproof/Spec.lean | 43 + .../v01_plane1_snapshot/lean/comparator.json | 19 + .../v01_plane1_snapshot/lean/lakefile.toml | 18 + .../p2b/v01_plane1_snapshot/spec/lineage.json | 139 + compile/src/mcproof_compile/hardware_probe.py | 24 +- docs/CONTINUITY.md | 16 +- docs/CROSS_POLLINATION.md | 56 + lean/Main.lean | 68 +- lean/Mcproof.lean | 3 + lean/Mcproof/AdapterBounds.lean | 66 + lean/Mcproof/Proofs.lean | 784 +- lean/Mcproof/Spec.lean | 188 +- lean/Mcproof/TransitionBounds.lean | 104 + lean/README.md | 29 +- lean/SPEC.md | 28 +- lean/bar.toml | 36 +- lean/comparator.json | 23 +- lean/tests/TEST-RESULTS.md | 4 +- lean/tests/run_tests.py | 94 +- spec/lineage.json | 15 +- src/mcproof/__init__.py | 54 +- src/mcproof/cli.py | 118 +- src/mcproof/domain_protocol.py | 1071 + src/mcproof/lean_harness.py | 695 +- src/mcproof/lineage.py | 2 + src/mcproof/mutation.py | 298 +- src/mcproof/oracles.py | 95 +- src/mcproof/repair_loop.py | 1396 + .../ux582hs_rtx3080_tc5_live_receipt.json | 613 + tests/test_domain_and_repair_loop.py | 459 + tests/test_lean_harness.py | 59 +- tests/test_lineage_prose_and_cli.py | 1 + tests/test_neon_audit_regressions.py | 285 + verification/VERIFICATION_RECEIPT.json | 6 +- verification/independent_rederive_p2b.py | 11 +- verification/verify_compiler_artifacts.py | 48 +- 43 files changed, 42730 insertions(+), 125 deletions(-) create mode 100644 compile/artifacts/evaluation/repair_loop/sealed_holdout_repair_loop_receipt.json create mode 100644 compile/artifacts/evaluation/repair_loop/validation_repair_b0.json create mode 100644 compile/artifacts/evaluation/repair_loop/validation_repair_p2b.json create mode 100644 compile/artifacts/evaluation/repair_loop/validation_repair_summary.json create mode 100644 compile/artifacts/p2b/v01_plane1_snapshot/lean/Mcproof.lean create mode 100644 compile/artifacts/p2b/v01_plane1_snapshot/lean/Mcproof/Proofs.lean create mode 100644 compile/artifacts/p2b/v01_plane1_snapshot/lean/Mcproof/Spec.lean create mode 100644 compile/artifacts/p2b/v01_plane1_snapshot/lean/comparator.json create mode 100644 compile/artifacts/p2b/v01_plane1_snapshot/lean/lakefile.toml create mode 100644 compile/artifacts/p2b/v01_plane1_snapshot/spec/lineage.json create mode 100644 docs/CROSS_POLLINATION.md create mode 100644 lean/Mcproof/AdapterBounds.lean create mode 100644 lean/Mcproof/TransitionBounds.lean create mode 100644 src/mcproof/domain_protocol.py create mode 100644 src/mcproof/repair_loop.py create mode 100644 stack/results/final/ux582hs_rtx3080_tc5_live_receipt.json create mode 100644 tests/test_domain_and_repair_loop.py create mode 100644 tests/test_neon_audit_regressions.py diff --git a/README.md b/README.md index b82560f..b8919c7 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,15 @@ `mcproof` is the verification harness, Plane 1 rulebook, dataset suite, weight compiler, and serving benchmark for the Siliconcordat Formal Verification MDE. It checks Lean 4.35.0 proofs across six kernels, grades exact-math checkers against planted bugs, compiles `weft` adapters, and audits live `warp-modeld` mounts. Proof authors, verifiers, and researchers use it to test formal reasoning without weakening deterministic gates. -Blueprint and gates live in [siliconcordat/mde issue 4](https://github.com/siliconcordat/mde/issues/4) under the [Master Plan](https://github.com/siliconcordat/.github/blob/main/MASTER-PLAN.md). Companion serving changes live in [siliconcordat/exo](https://github.com/siliconcordat/exo/tree/feat/mde4-phase4-phase6-stack) and [siliconcordat/warp PR 5](https://github.com/siliconcordat/warp/pull/5). +Blueprint and gates live in [siliconcordat/mde issue 4](https://github.com/siliconcordat/mde/issues/4) under the [Master Plan](https://github.com/siliconcordat/.github/blob/main/MASTER-PLAN.md). Companion serving changes live in [siliconcordat/exo](https://github.com/siliconcordat/exo/tree/feat/mde4-phase4-phase6-stack) and [siliconcordat/warp](https://github.com/siliconcordat/warp/tree/feat/mde4-phase4-live-gate). ## Unified Repository Layout | Path | Contents | | --- | --- | -| `src/mcproof/`, `bin/mcproof`, & `bin/bootstrap-deps` | Python 3.11+ Tier 2 oracle runner (`oracles.py`, `lean_harness.py`, `mutation.py`, `lineage.py`, vendored `prose_gate.py`, `cli.py`) and pinned external checkout bootstrap | -| `docs/CONTINUITY.md` | Capability steward record (`mde:formal-verification`), three-tier contracts, scratch reproduction runbook, and empirical Phase 1 to 6 lessons | -| `lean/` | Stock-`Init` Lean `v4.35.0` reference library, challenge/solution modules, 190-case differential test suite, and 6-kernel `lake comparator --paranoid` config | +| `src/mcproof/`, `bin/mcproof`, & `bin/bootstrap-deps` | Python 3.11+ Tier 2 oracle runner, pluggable `DomainOracle` protocol, label-free `repair_loop.py`, `F-01..F-08` certificate auditor, vendored `prose_gate.py`, and CLI | +| `docs/CONTINUITY.md` & `docs/CROSS_POLLINATION.md` | Capability steward record (`mde:formal-verification`), bidirectional `mcproof`/`soko`/collaborator transfer ledger, and Phase 1 to 6 lessons | +| `lean/` | Stock-`Init` Lean `v4.35.0` reference library (`29` theorems across `Mcproof.lean`, `TransitionBounds.lean`, and `AdapterBounds.lean`), `370`-case differential suite, and `6`-kernel `lake comparator --paranoid` config | | `spec/` & `schemas/` | Audited `32,216`-token (`Qwen3.5-4B`, `30,873` `cl100k_base`) Plane 1 specification (`plane1_formal_spec.md`), SHA-256 `lineage.json`, and canonical `zone-manifest.v0.1.json` | | `datasets/` | Frozen public `compile.jsonl` (`96`), `validation.jsonl` (`48`), `hybrid_k0.jsonl` (`2,048` noise + `24` `y=0` anchors), `144` gold targets, `scripts/grade_item.py`, and `SHA256SUMS` (`312953bc...`) | | `commitments/` | Public-safe cryptographic commitment (`PUBLIC_HOLDOUT_COMMITMENT.json`, zero prompt text) binding the `242`-item sealed holdout and canary vault | @@ -42,7 +42,14 @@ On public `validation.jsonl` (`48` items), `P2-B` regressed below bare `B0` and | Rollback fidelity | `0.0` next-token logprob diff after `O(1)` kill switch | `0.0` exact top-20 logprob diff (`5.528 ms` multi-slot flush) | **PASS** | | Phase 6 `T-C1` solo preservation | `>= 90%` live behavioral preservation in paired/triple stacks | `0.0%` min live `step0` retention (`weft-eido` `0.0611` live drift vs `0.0019` solo shift, `99.85%` CPU subspace diagnostic) | **FAIL (Live `step0` Drift)** | | Phase 6 `T-C3`, `T-C4`, `H1`-`H7` | Base retention, order invariance, and 7 live hazards | `85.0%` vs `84.0%` `W0` (`T-C3`), `0.0` perm/rollback diff (`T-C4`, `4.602 ms`), `7/7` hazards | **PASS** | -| Phase 6 `T-C5` serving budget | `<= 15%` overhead (`>= -5%` floor), `>= 8k` KV, `<= 7.2 GiB` NVML VRAM | `-33.1%` same-server latency (`benchmark_anomaly = true`), `8192` KV/slot, analytical `4.98 GiB` VRAM (no 8 GiB NVML GPU online) | **FAIL (Timing Anomaly & Analytical VRAM)** | +| Phase 6 `T-C5` serving budget | `<= 15%` overhead (`>= -5%` floor), `>= 8k` KV, `<= 7.2 GiB` NVML VRAM | Live `UX582HS` RTX 3080 (`-c 65536 -np 8`): `Q8_0` `6.8525 GiB`, `1`-slot `+0.38%`, `8`-slot `-0.66%` (`Q4_K_M` `5.2314 GiB`, `+0.47%` / `+0.48%`) | **PASS (v0.2 Live NVML)** | + +### v0.2 Closed-Loop Plane 1 Retrieval and Tier 2 Repair ([Proven-Commons/mcproof issue 5](https://github.com/Proven-Commons/mcproof/issues/5)) + +Under `100%` label-free `default_inference_oracle` stopping (`<= 1,500`-token Plane 1 retrieval + AST/token state-hash + `seen_fail_sigs` stagnation escape, receipts in `compile/artifacts/evaluation/repair_loop/`): + +- **Public `validation.jsonl` (`48` items)**: `B0` climbed from `4/48` (`8.33%` open-loop) to `5/48` (`10.42%`, `k=1`) to `6/48` (`12.50%`, `k=2`) to **`8/48` (`16.67%`, `k=4`, `2.0x` open-loop)** (`A: 1/12, B: 5/12, C: 0/12, D: 2/12`), while single-layer `blk.31` `P2-B` scored `0/48` (`67` stagnation events). +- **Sealed `private_holdout.jsonl` (`96` items)**: broke the `0/96` single-pass ceiling across both arms (`B0` `0/96 -> 1/96` [`HOLD-C-04` at `k=2..4`], `P2-B` `0/96 -> 1/96` [`HOLD-D-10` at `k=3..4`]). ## Prerequisites and Fresh-Clone Reproduction @@ -87,11 +94,11 @@ Run the full Tier 2 oracle gates and Family `A` to `D` graders. On Linux, `lake - Passing Tier 2 oracles proves only the machine-checked properties enforced by each checker. Whether a theorem statement matches human intent remains a Tier 1 human specification judgment. - `datasets/scripts/grade_item.py` grades Family C items by deterministic structural claim-tier and `meta["bar_status"]` replay rather than re-running Lean compilation on prose-only reports. -- Live GPU runs executed on a local 64 GiB Apple M1 Max (`llama-server 0.5.0`, `$0` cloud). Analytical 8 GiB RTX 3080 VRAM projections (`4.9753 GiB` Linux headless / `6.1753 GiB` Windows WDDM on `Q8_0`) remain unmeasured on live 8 GiB NVML hardware (`vram_measurement_kind = analytical_projection_not_nvml`). +- Live `8` GiB RTX 3080 `T-C5` telemetry (`stack/results/final/ux582hs_rtx3080_tc5_live_receipt.json`) measured Windows 11 NVIDIA Optimus (`driver 566.07`, `0 MiB` idle WDDM VRAM), while bare-metal Linux kernel execution on `UX582HS` remains tracked separately. ## What you must trust -- Lean `v4.35.0` and its five bundled external kernels (`leanchecker-paranoid`, `lean4lean`, `nanoda`, `con-leche`, `con-ron`), plus the Lean compiler and C runtime for native binaries (`Classical.choice`, `Quot.sound`, and `propext` measured via `collectAxioms` on `Mcproof.Proofs`). +- Lean `v4.35.0` and its five bundled external kernels (`leanchecker-paranoid`, `lean4lean`, `nanoda`, `con-leche`, `con-ron`), plus the Lean compiler and C runtime for native binaries (`Quot.sound` and `propext` measured via `collectAxioms` across all `29` reference theorems, with zero `Classical.choice`). - Python 3 exact `int` and `fractions.Fraction` arithmetic (`-I -O -B -W error`), `numpy` CPU `float64` linear algebra, and the Rust `tree-sitter` CST parser in `siliconcordat/mde`. - Host OS sandboxing (`bwrap` network-unshared read-only root on Linux, or subprocess `-I` environment isolation without OS syscall confinement on macOS and Windows) and SHA-256 digest implementations. diff --git a/compile/artifacts/evaluation/repair_loop/sealed_holdout_repair_loop_receipt.json b/compile/artifacts/evaluation/repair_loop/sealed_holdout_repair_loop_receipt.json new file mode 100644 index 0000000..7b7c799 --- /dev/null +++ b/compile/artifacts/evaluation/repair_loop/sealed_holdout_repair_loop_receipt.json @@ -0,0 +1,22682 @@ +{ + "arms": { + "B0": { + "arm": "B0_repair_loop", + "budget_by_k": { + "k=1": { + "completion_tokens": 53775, + "mean_retrieved_plane1_tokens_per_turn": 1200.0, + "oracle_calls": 96, + "prompt_tokens": 119806, + "replies_used": 96, + "total_tokens": 173581 + }, + "k=2": { + "completion_tokens": 106485, + "mean_retrieved_plane1_tokens_per_turn": 1203.9, + "oracle_calls": 183, + "prompt_tokens": 286816, + "replies_used": 183, + "total_tokens": 393301 + }, + "k=4": { + "completion_tokens": 205658, + "mean_retrieved_plane1_tokens_per_turn": 1201.5, + "oracle_calls": 339, + "prompt_tokens": 574098, + "replies_used": 339, + "total_tokens": 779756 + } + }, + "by_family": { + "A": { + "k=1": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=2": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=3": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=4": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + } + }, + "B": { + "k=1": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=2": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=3": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=4": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + } + }, + "C": { + "k=1": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=2": { + "deployed_passed": 1, + "total": 24, + "trajectory_any_passed": 1 + }, + "k=3": { + "deployed_passed": 1, + "total": 24, + "trajectory_any_passed": 1 + }, + "k=4": { + "deployed_passed": 1, + "total": 24, + "trajectory_any_passed": 1 + } + }, + "D": { + "k=1": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=2": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=3": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=4": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + } + } + }, + "deployed_pass_curve": { + "k=1": { + "pass_rate_pct": 0.0, + "passed": 0, + "total": 96 + }, + "k=2": { + "pass_rate_pct": 1.04, + "passed": 1, + "total": 96 + }, + "k=3": { + "pass_rate_pct": 1.04, + "passed": 1, + "total": 96 + }, + "k=4": { + "pass_rate_pct": 1.04, + "passed": 1, + "total": 96 + } + }, + "lora": [], + "outcomes": [ + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-01", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2953, + "total_elapsed_ms": 248088.53, + "total_prompt_tokens": 6618, + "total_retrieved_plane1_tokens": 4782, + "total_tokens": 9571, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 712, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1195, + "response_sha256": "67088c9bb7950175ba4c2bfbbadd1390b9db8859f0a76c7bc576d2cf95307a83", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 705, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2003, + "response_sha256": "30cbae640194f89d43e698b6447c06e058664ec8e5498fede3b9679ecb3d4485", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1987, + "response_sha256": "17430c60605044e0829a718b580c8ef2105a8cb5525e2d234ef9d8cc048f5eb9", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1433, + "response_sha256": "c029f58ff28dfb104793e77d1c5b0e36329a23eec1b9a4727ed0aeede98cdcdb", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_70", + "sec12_78" + ], + "retrieved_plane1_tokens": 1170, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-02", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1881, + "total_elapsed_ms": 159119.84, + "total_prompt_tokens": 6212, + "total_retrieved_plane1_tokens": 4590, + "total_tokens": 8093, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 368, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1205, + "response_sha256": "45f16a6820eacdf698d4b5549f68393f8a77d6f787340f3908ae0a692893339a", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 407, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1745, + "response_sha256": "f47797096ef0fa0049760287f23054ee004e2243daba001747522ef10fbd10ee", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 338, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1790, + "response_sha256": "71f868dcbfd2ea7b899ceae5da50354bac222a5657cbad2b896253a6057f6e1d", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1472, + "response_sha256": "92b8c9930a6693fc5dbb1f573c42725663828921e585a7fc5c49dc634889b19f", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_70", + "sec12_78" + ], + "retrieved_plane1_tokens": 1170, + "sealed_error_categories": [ + "missing_trust_words", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-03", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 253299.72, + "total_prompt_tokens": 7261, + "total_retrieved_plane1_tokens": 4576, + "total_tokens": 10333, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1235, + "response_sha256": "cb9c779641fe2fe74bc823c1a7254fb4c156f5b72bfa896ff38fdee05c584ebc", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2102, + "response_sha256": "dc8d1dc771655b3a631a458bbb1de2f15854e99eb5532c1b64b744845e9b5167", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1909, + "response_sha256": "8cf60a2cbd70930819208f8d60e7530404deaff4f729a09edc6c5c25ebb6bb5a", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2015, + "response_sha256": "0887a19e86183f2d043b2f46ffd9ed840e81aaa5368ae45d8e4bcb76ac7564de", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-04", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 2584, + "total_elapsed_ms": 219356.62, + "total_prompt_tokens": 6920, + "total_retrieved_plane1_tokens": 4769, + "total_tokens": 9504, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 606, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1149, + "response_sha256": "ae8f533e80ff6adc039a0959b48d124ac9741247f57985060f175a2ad2b3a31e", + "retrieved_chunk_ids": [ + "sec12_78", + "sec4_09", + "sec4_14", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1195, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 484, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1949, + "response_sha256": "94ec41faceaaa6ef7c58dd64b7bae0ef1536c3798d67b3d9a6c52adae5816351", + "retrieved_chunk_ids": [ + "sec13_79", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1217, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 726, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1851, + "response_sha256": "4a8881638b5b2c2c675ac32287c26731c933483964004214e524d2425dadf8c4", + "retrieved_chunk_ids": [ + "sec13_79", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1217, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1971, + "response_sha256": "17b9da7848b70177d7fb7a6aa47920d560c8d4dc0d2b10406201564b9352798f", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-05", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 2907, + "total_elapsed_ms": 245613.92, + "total_prompt_tokens": 7438, + "total_retrieved_plane1_tokens": 4696, + "total_tokens": 10345, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 684, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1228, + "response_sha256": "82d35dbd6ef4dc68ab2bc03dac8114748f93a0388ca718161a8a103176747a81", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 726, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "UNVERIFIED_NATIVE_DECIDE", + "UNVERIFIED_NATIVE_EVAL", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2093, + "response_sha256": "3caa33080baae8382ca598b7491b35c6d47b08d83a62e45df941a1fa77f38b9c", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 729, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2073, + "response_sha256": "c3dc60ac67f13f06a2abb0cc57ca3ffe568f403dcf6dd8acee5ad835a133d855", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "EXTRA_AXIOM_DECL", + "UNVERIFIED_NATIVE_DECIDE", + "UNVERIFIED_NATIVE_EVAL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2044, + "response_sha256": "ad1e48753505a9594ffc3bcadfcf684a2154dcad3b4a58ce1886e5d84dc997ad", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-06", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 2937, + "total_elapsed_ms": 247278.44, + "total_prompt_tokens": 7552, + "total_retrieved_plane1_tokens": 4624, + "total_tokens": 10489, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1265, + "response_sha256": "c9e39bf2247bfbfd31cc46f7fe613b84748532b5791771f64cb47582810692ba", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 767, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2142, + "response_sha256": "8a4f04c27e8def94927c126b4125f2f189f322b89da480b1a26950eb7e761870", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "EXTRA_AXIOM_DECL", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2045, + "response_sha256": "093bce485a09696f2f4e20867b54e823ce5280b9e5537cb5bf4eda347bd57f68", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 634, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2100, + "response_sha256": "d9835c81646a44f0532e537aff9458e1f2bb3cd1d4f15331c1c9fe63b31bba5a", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-07", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 2364, + "total_elapsed_ms": 204239.58, + "total_prompt_tokens": 7114, + "total_retrieved_plane1_tokens": 4624, + "total_tokens": 9478, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 519, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1251, + "response_sha256": "13e67395b3f6157332508f4f64084ee4f4413dbe345bc27c34880d002396fcf7", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_trust_words", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 615, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1890, + "response_sha256": "6d79cb04285e89a07607b0cad482622a69cc327b5771450b5c4ca324cd49a177", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "EXTRA_AXIOM_DECL", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1979, + "response_sha256": "81eb61ef50b634af142b2021b86b744741311fe561afcdd59ef49d6969bcfc95", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 462, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1994, + "response_sha256": "6a8089658defa8ffb9a55cdd72d5b8dba93aead8cab8aa63d63d4edb55ec833b", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-08", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 253337.45, + "total_prompt_tokens": 7684, + "total_retrieved_plane1_tokens": 4624, + "total_tokens": 10756, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1281, + "response_sha256": "fe4a752ca36078520a8e93c8d1c8cb8cc3b0cca5d46a1171c4503b2793be6978", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2098, + "response_sha256": "49fe4fe78ef310f04688b1e9a53b3699a3083f3cec6a8a1d1537ab41a4271ddf", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2189, + "response_sha256": "858b5151acd5895aa37db82c984d1814a9dfc9df4b2dab15f04d8e0152d6ba2b", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "FORBIDDEN_META_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2116, + "response_sha256": "fb85b03ab298d1d62f34d4c993d172d45b0bfd93afdd7d8bbad92de16fd4ee3b", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-09", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 242576.25, + "total_prompt_tokens": 7352, + "total_retrieved_plane1_tokens": 4648, + "total_tokens": 10424, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1298, + "response_sha256": "cb9883f64e287550c055c0ae063e80be2ee5dd99596db93ace438ceeb9372e3c", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2199, + "response_sha256": "378d7c975bfdcff56c1935df3f0281985227cdaadc9cb53ffbd0c364d3f0e544", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2217, + "response_sha256": "378d7c975bfdcff56c1935df3f0281985227cdaadc9cb53ffbd0c364d3f0e544", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1638, + "response_sha256": "0a52f9577c8f2cab4acd05e3b45b0f908cd5e65917b68fa50fdc309a29f79d47", + "retrieved_chunk_ids": [ + "sec12_76", + "sec12_78" + ], + "retrieved_plane1_tokens": 1132, + "sealed_error_categories": [ + "missing_trust_words", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-10", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2925, + "total_elapsed_ms": 233733.29, + "total_prompt_tokens": 7235, + "total_retrieved_plane1_tokens": 4770, + "total_tokens": 10160, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 621, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1291, + "response_sha256": "c406da0cadbee180f4a05617555e3110384e25fb5ac9dfc0666d4b90f1a932a5", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2109, + "response_sha256": "568ea1f51bdae1b70cc4efae7197bab2576427e2483fa04eff7ef62585b2fdc3", + "retrieved_chunk_ids": [ + "sec13_79", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1217, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2192, + "response_sha256": "d8f17229fb44781c26b95447aef1c279a19a72fd327dc4984bc606ae9df3cc6d", + "retrieved_chunk_ids": [ + "sec13_79", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1217, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "FORBIDDEN_META_IMPORT", + "FORBIDDEN_META_IMPORT", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1643, + "response_sha256": "f6c925a85ca78a54b60298559ef1757dd0017d3a3ca65881a9f17daddad22c29", + "retrieved_chunk_ids": [ + "sec12_76", + "sec12_78" + ], + "retrieved_plane1_tokens": 1132, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-11", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 242246.35, + "total_prompt_tokens": 7838, + "total_retrieved_plane1_tokens": 4693, + "total_tokens": 10910, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1265, + "response_sha256": "26cfe6f23624b4577f947ea17aeb4f3145e84b317dd19c0dcd20664c0f399c35", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec12_78", + "sec4_09", + "sec9_44", + "sec9_48", + "sec9_55" + ], + "retrieved_plane1_tokens": 1147, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "EXTRA_AXIOM_DECL", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2212, + "response_sha256": "108ffe44b493519d21de4a6bab338a26790c3fd64d8b6efbe1b3a3a6c840cb99", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_55" + ], + "retrieved_plane1_tokens": 1171, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2196, + "response_sha256": "3902ef93f9161aeaee5c1bd5ade02ccf9da56426613a8fbc2d1f2333ad65a212", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2165, + "response_sha256": "3902ef93f9161aeaee5c1bd5ade02ccf9da56426613a8fbc2d1f2333ad65a212", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_55" + ], + "retrieved_plane1_tokens": 1171, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-12", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 1651, + "total_elapsed_ms": 144048.43, + "total_prompt_tokens": 6140, + "total_retrieved_plane1_tokens": 4654, + "total_tokens": 7791, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 329, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1194, + "response_sha256": "7134022f2ab963d557aa446b578cc8e90eaf0c0e61fa1dd8eba54025ea7d70db", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_trust_words", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 659, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1655, + "response_sha256": "e096c45dfa834a3bcf91c57748e58701225669446f50113855450b2ec97dcfe5", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 353, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1864, + "response_sha256": "6a4c4a178e6d7f31075f317c43422edb6a4e01c42c8bf5e84bd932ca0487dd57", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 310, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1427, + "response_sha256": "f5260d681ff0795cc6df72d460aebdc33a661794f7946d3430b5c857b2483216", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_70", + "sec12_78" + ], + "retrieved_plane1_tokens": 1170, + "sealed_error_categories": [ + "missing_trust_words", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-13", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 241934.8, + "total_prompt_tokens": 7610, + "total_retrieved_plane1_tokens": 4538, + "total_tokens": 10682, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1234, + "response_sha256": "9bb6f1e08ee05c1a006243102abc28779d2fa9f2ca657a940f1cc8f1fe57a64f", + "retrieved_chunk_ids": [ + "sec12_78", + "sec4_12", + "sec9_42", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1162, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2175, + "response_sha256": "cf11633e6b7665dea37eb69fb835a684511631225790a4e19caac5e99eca6e54", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec9_54" + ], + "retrieved_plane1_tokens": 1096, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "EXTRA_AXIOM_DECL", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2175, + "response_sha256": "c78ba13351593a8de8a6359f6c0f986704ee9e846db60b31a9fdc75aedfc5a6f", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "UNPROVED_SORRY" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2026, + "response_sha256": "33dc3e70325d1e5de437ce01edf7f2704b40001613616d642f225e5945bb6dd1", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-14", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3059, + "total_elapsed_ms": 241634.4, + "total_prompt_tokens": 7685, + "total_retrieved_plane1_tokens": 4672, + "total_tokens": 10744, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 755, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1311, + "response_sha256": "e8199acbfaf1b5a7e9bbd3644a724754fea1d8cb353b89c57e39df62cb314a10", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "EXTRA_AXIOM_DECL", + "UNVERIFIED_NATIVE_DECIDE", + "UNVERIFIED_NATIVE_EVAL", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2152, + "response_sha256": "c24362d1d90596e47c217f595b4b92de3dcf6006318b37af03ecc104ab6305e9", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2145, + "response_sha256": "a046f9d9a93047e2683ba51c646c59cc37cf98dd24c952653793cc8e8a1a3513", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "EXTRA_AXIOM_DECL", + "UNVERIFIED_NATIVE_DECIDE", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "UNVERIFIED_NATIVE_EVAL", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2077, + "response_sha256": "d8048a719417f29d19928ef3d41185d5b805065b3a7f0ac400828dba3339787d", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-15", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2640, + "total_elapsed_ms": 219481.88, + "total_prompt_tokens": 6936, + "total_retrieved_plane1_tokens": 4755, + "total_tokens": 9576, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 677, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1226, + "response_sha256": "acddec0122cd92d186ef4f59d170670ac990f08d0e7bede470cd85e4382aa9ca", + "retrieved_chunk_ids": [ + "sec12_78", + "sec4_07", + "sec4_09", + "sec9_46" + ], + "retrieved_plane1_tokens": 1210, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 618, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2152, + "response_sha256": "25408d223a04655514537a99c0092e4fe18399ba0cf1285ed7a55308ec5b7216", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1195, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 577, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2021, + "response_sha256": "0609782fc9407123e1d93f30799959e64eb2130a0cf93209abf3df01cc06d310", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1537, + "response_sha256": "df6b1a2f5321dc900e71726452d0bb9b12d2689e88a19626aeeda830821fd836", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_70", + "sec12_78" + ], + "retrieved_plane1_tokens": 1170, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-16", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 2677, + "total_elapsed_ms": 222270.06, + "total_prompt_tokens": 7365, + "total_retrieved_plane1_tokens": 4735, + "total_tokens": 10042, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 550, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1136, + "response_sha256": "60a89cde482e84e44edaf5f71fd703f4f6d2d9e9a354b718296b7206cc218917", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_78", + "sec4_07", + "sec4_09", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1195, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 591, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2060, + "response_sha256": "627d0747e385da050f572dfe36d8ff903a02d7581464a7bbf63ea411ffe7598f", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "EXTRA_AXIOM_DECL", + "UNVERIFIED_NATIVE_DECIDE", + "UNVERIFIED_NATIVE_EVAL", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1950, + "response_sha256": "cdba4feb4f97c591ee2e5d45feb563fbb382c98920888166623db1eb25e9ae07", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2219, + "response_sha256": "e96af3d43484e0b133fe982d7caa43a8666f69a2152d3976aaef339d2fd892e2", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-17", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 2936, + "total_elapsed_ms": 232650.92, + "total_prompt_tokens": 7707, + "total_retrieved_plane1_tokens": 4624, + "total_tokens": 10643, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 632, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1311, + "response_sha256": "5396f61e07094f6f07e875df829a96aeb0db820b4f5e92da964ad9d741c9dd74", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2189, + "response_sha256": "db08df164f234fa6f397869c601bcbddf8fe417ff17a01101057b6ed9b40df09", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2101, + "response_sha256": "71a348a2477020e5a29496b23d873273fe8477c94a49a094c0d3184f731fac3b", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2106, + "response_sha256": "790332c29839867283d54e76a35d2610cce5569da3fe1f1fdd101200497e3e40", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-18", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 3002, + "total_elapsed_ms": 235056.25, + "total_prompt_tokens": 6394, + "total_retrieved_plane1_tokens": 4680, + "total_tokens": 9396, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1261, + "response_sha256": "8aa3ca6ab6533d404d2eb283252d355c157047bd96c5ee03cbd462a7779aa2c3", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_74", + "sec4_09", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1174, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2007, + "response_sha256": "c7d5415b5c33caf7d12ab22a34fc9e9ad027de6545c61592a8847265ef81fd96", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_74", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1175, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1521, + "response_sha256": "20950694ba329d0f2e2a8857d8d7382a2a60c0e641c5e97d545e777ac1998a4d", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_70", + "sec12_78" + ], + "retrieved_plane1_tokens": 1170, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 698, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1605, + "response_sha256": "25f4653a9d6064cb511d943271d64bdb5a73b10d42f7eece928ffe19a8883b66", + "retrieved_chunk_ids": [ + "sec12_71", + "sec9_49" + ], + "retrieved_plane1_tokens": 1161, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-19", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 239253.94, + "total_prompt_tokens": 7979, + "total_retrieved_plane1_tokens": 4710, + "total_tokens": 11051, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1308, + "response_sha256": "588a8d97a8233e03b4ada34d1e69a28fa14bba265a11654adbad0bdacd7ecf0e", + "retrieved_chunk_ids": [ + "sec12_78", + "sec4_12", + "sec9_42", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1162, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "EXTRA_AXIOM_DECL", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2317, + "response_sha256": "1100d1bb098bcdd12ee48d00884b49938093a53ffea2d12259de4d5b44c9f445", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "FORBIDDEN_META_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2148, + "response_sha256": "ec8c3004d66c53179e8510b89c7481ec080b296ca0bfefa17d35e5e0c036e2e8", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2206, + "response_sha256": "1dc80e855b5bea279bf4894ab18f849fa1645564903cff59b33e3e61456636ef", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-20", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 238613.79, + "total_prompt_tokens": 7361, + "total_retrieved_plane1_tokens": 4663, + "total_tokens": 10433, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1340, + "response_sha256": "d716916fbe628d354d5793082e194a8389032a5618be2184d36eadf855b51cf4", + "retrieved_chunk_ids": [ + "sec12_78", + "sec4_12", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1144, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2278, + "response_sha256": "6390d1766356ec42de84fd7cd70afe3d4ea0661bd90981696368c98da2f8e1af", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2182, + "response_sha256": "6390d1766356ec42de84fd7cd70afe3d4ea0661bd90981696368c98da2f8e1af", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1561, + "response_sha256": "2b22a832a470e416fe06f1ff40a21f69ef76b1d163c4849af08be74b52073cb9", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_70", + "sec12_72", + "sec12_74", + "sec12_78", + "sec4_09" + ], + "retrieved_plane1_tokens": 1159, + "sealed_error_categories": [ + "missing_trust_words", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-21", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 239381.39, + "total_prompt_tokens": 7905, + "total_retrieved_plane1_tokens": 4790, + "total_tokens": 10977, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1333, + "response_sha256": "b74543fa8283cb2bf18859d0ac0c4f25e7591655a8185af6406d8d7d7948e987", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_78", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1213, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "EXTRA_AXIOM_DECL", + "OPAQUE_CONST_DECL", + "UNVERIFIED_NATIVE_DECIDE", + "UNPROVED_SORRY", + "UNSOUND_IMPLEMENTED_BY" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2291, + "response_sha256": "148eb8dba03cbea7ae9cf09d9e5437dbfe5c29fa2b9ae5959850f178e9e81ede", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_trust_words", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2070, + "response_sha256": "93675080583ada509e38b885ba414d5ffd232a468ce89c9f719337b4f7e20c77", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec4_09", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1193, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2211, + "response_sha256": "66627654de5fa3361fa7daa7aa870c5c999ffc1b13c02067731250ec0716e977", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-22", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2866, + "total_elapsed_ms": 228795.43, + "total_prompt_tokens": 7663, + "total_retrieved_plane1_tokens": 4705, + "total_tokens": 10529, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 562, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1317, + "response_sha256": "6207be766758166682bffeaaec57490ab424c01a5f89d436d4d725a0259a8079", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_78", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1213, + "sealed_error_categories": [ + "missing_trust_words", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1970, + "response_sha256": "14709089cdb608014a8a0579ee9e7273e378776cf600f16bb65f66a8cf222130", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2209, + "response_sha256": "e13ea316d3e6f13895fa45d2bbf209de494484cf97752b00b9fd4e9e67da4428", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2167, + "response_sha256": "3a883cfeb2019a8fcf865c772d453d6ba6ae831425dbf1b604ac02d1378d9ebe", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-23", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 238690.94, + "total_prompt_tokens": 7410, + "total_retrieved_plane1_tokens": 4624, + "total_tokens": 10482, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1278, + "response_sha256": "6a2f1ce9957fd7ba049be22a8dcd571dc8e695a3e6be5b1f2b5b0326ad812e7e", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2039, + "response_sha256": "8d15aec127d74592278d5b199f8d3daa58ee672a1a55306577099debcb1bfcb7", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2031, + "response_sha256": "f9b1956ddbd283c131c3b5fcb46c606c0837878ff9d5195342dce03fd4694ce1", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2062, + "response_sha256": "ff74fdba10102f34349354e2c79c72a6392b0b06316990904b0810aaebad246c", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_trust_words", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-24", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2721, + "total_elapsed_ms": 222322.47, + "total_prompt_tokens": 6895, + "total_retrieved_plane1_tokens": 4728, + "total_tokens": 9616, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 685, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1275, + "response_sha256": "4546f2c65b49925f064394eafe57405f4b3b4850181144a770c475ee45c88760", + "retrieved_chunk_ids": [ + "sec12_78", + "sec4_09", + "sec4_12", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1152, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 699, + "inference_diagnostic_rules": [ + "FORBIDDEN_META_IMPORT", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2094, + "response_sha256": "3df12284b93fae5e1bef402c35e2128eb924a67712941264352ddac27fc3e8e5", + "retrieved_chunk_ids": [ + "sec12_66", + "sec9_42", + "sec9_43", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1203, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 707, + "inference_diagnostic_rules": [ + "FORBIDDEN_META_IMPORT", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2031, + "response_sha256": "10d8dce0789e563efe84ba74bc80545249746c14161d10a9b5f88cb16ece800e", + "retrieved_chunk_ids": [ + "sec12_66", + "sec9_42", + "sec9_43", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1203, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 630, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1495, + "response_sha256": "2769a6f9647809a6a61028eca759980bb11626247c064d974ea9d288c61ea539", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_70", + "sec12_78" + ], + "retrieved_plane1_tokens": 1170, + "sealed_error_categories": [ + "missing_trust_words", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-01", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 768, + "total_elapsed_ms": 54434.22, + "total_prompt_tokens": 1299, + "total_retrieved_plane1_tokens": 1206, + "total_tokens": 2067, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1299, + "response_sha256": "b6fa1285d94652dbfd93f1002f72d8b4651e06373b99f83f593d5a9d3eeee88e", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "mutation_kill_below_floor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-02", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 219174.27, + "total_prompt_tokens": 7175, + "total_retrieved_plane1_tokens": 4863, + "total_tokens": 10247, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1239, + "response_sha256": "798c20a86300d1d3fa8d62a3c12b54eb22e159658acb04123a68b6bd3e571c41", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2023, + "response_sha256": "210c506513640be4f0dd41844bc5c12a40647ae4f113c18c6a37e069935a457e", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1951, + "response_sha256": "823fd7144dfdf57aa5508754437ec1bcd74eba897f3132e4406aa80443bc9fa7", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1962, + "response_sha256": "e79fb71fc91161402561a84f159fb6d14dd4344e2af6157f20bd348198368083", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-03", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 219142.91, + "total_prompt_tokens": 7430, + "total_retrieved_plane1_tokens": 4863, + "total_tokens": 10502, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1274, + "response_sha256": "d0ae48efc1b1ee324ca7df66323b4e987416a490bcc4d6a39abafd63bd0fcf6d", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2108, + "response_sha256": "2a49016c0d9bd22a3ed9d29f77e21730d42bad8c5dc78856df2ba550a20612dd", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1963, + "response_sha256": "12ac636c7d1f13db84e43e7d6efd5fd54ea94daa19af123ee3aef6514076b137", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2085, + "response_sha256": "13d41a3c6667c8ff81d861c20928e2b4ebcf57e49fed9813bf48aeb2667c7657", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-04", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 768, + "total_elapsed_ms": 56142.42, + "total_prompt_tokens": 1347, + "total_retrieved_plane1_tokens": 1204, + "total_tokens": 2115, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1347, + "response_sha256": "15bb25ca2822c33ec2cb977f85967077a46edd63421c6afd0414344166c576e6", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-05", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 219246.13, + "total_prompt_tokens": 6178, + "total_retrieved_plane1_tokens": 4848, + "total_tokens": 9250, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1272, + "response_sha256": "3d48929117b16b613aff91ead400bf2be8b03ccde8f573530fd8a6f994c6a98b", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec6_25", + "sec6_26", + "sec6_30" + ], + "retrieved_plane1_tokens": 1202, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2005, + "response_sha256": "a275ac02c1e756bd879c181344cf5a4ceb31c1b9509132d32824ef9f05cc91c8", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1480, + "response_sha256": "ea516d2bcb38a5b88227938cb8ab79d0d0122c4c907ca6664fb501081750dded", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_69", + "sec12_72", + "sec5_18", + "sec5_20" + ], + "retrieved_plane1_tokens": 1209, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1421, + "response_sha256": "0c8f9c958935f1799f2140ca446d20d68086ff8ec8a6c6654b4370e2026fc7fa", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1218, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-06", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 219271.45, + "total_prompt_tokens": 6977, + "total_retrieved_plane1_tokens": 4840, + "total_tokens": 10049, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1456, + "response_sha256": "0e444acbfc57f46b8aabcb347a1aa9f6d63b3b9f00d069fe72651c1558e62073", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2080, + "response_sha256": "d96c68277e476f59f01336baa71f43b28263e609d64a4765156601f3219738c0", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1683, + "response_sha256": "1700c447088241825ce066e6b6c47d74052ad8f6be7aa23a8fd0a7d8ffc2ba30", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1758, + "response_sha256": "3818f0e1a1217805825733d1846960dcdcd6e246ddc38e30e88d8a4c0472641d", + "retrieved_chunk_ids": [ + "sec12_72", + "sec12_74", + "sec5_20", + "sec6_30" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-07", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 219306.53, + "total_prompt_tokens": 6313, + "total_retrieved_plane1_tokens": 4840, + "total_tokens": 9385, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1290, + "response_sha256": "89d2eabee8259a9acdd5c3ca378e5fa0a4f654d4c50eef08d6a305eae43cb612", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1907, + "response_sha256": "b9cf80cf1bd4539f254ca9de0874dfecd606e891f506657f4287ac40ba087f42", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1521, + "response_sha256": "127171cb0c91d2511c004ec194ed58bdec6146d82a83ebe755846ad3b7796ec7", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1595, + "response_sha256": "bc2d7bf9dcf4217f97c50e4c7a42e8386411c603e2255f05bf4d62a85c599891", + "retrieved_chunk_ids": [ + "sec12_72", + "sec12_74", + "sec5_20", + "sec6_30" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-08", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 219331.3, + "total_prompt_tokens": 6296, + "total_retrieved_plane1_tokens": 4842, + "total_tokens": 9368, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1298, + "response_sha256": "f5716b5c6106164a4e7b10d238d5e6c2f005c7948567d78ce5cf9f3c9fbcdd97", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec6_25", + "sec6_26", + "sec6_30" + ], + "retrieved_plane1_tokens": 1202, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1931, + "response_sha256": "4879b427d15c9dab001c28b00ca8a18c8f9094f691b43fad3a80b05cec8683e8", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1527, + "response_sha256": "75ac8ba1eaed82b3f5385b41357029b9e69fe6716e7a979377bdc4e6ae34f389", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1540, + "response_sha256": "ada0662f5e2dbb5369c0b734a4dd017306c7f7cec12a27d3009417cf531e6134", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_72", + "sec12_74", + "sec5_18", + "sec5_20" + ], + "retrieved_plane1_tokens": 1213, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-09", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 239868.15, + "total_prompt_tokens": 6228, + "total_retrieved_plane1_tokens": 4846, + "total_tokens": 9300, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1278, + "response_sha256": "28939647fd0b7bb47ebbeba615c8d78c8d4e108a5dee67af5ff57de3e7429ef7", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1981, + "response_sha256": "a09627f286bff48b6843af874cb0364f72910c1aea4b0d8b9e59702ed5a3c723", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1547, + "response_sha256": "9acc27c0dd89bc72426c90f746ab63e90b0309a3ca50f6c9c46463311addb697", + "retrieved_chunk_ids": [ + "sec12_69", + "sec12_72", + "sec5_20", + "sec6_30" + ], + "retrieved_plane1_tokens": 1203, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1422, + "response_sha256": "5df47c824354b0564afc864470858ef1d7b402b8890cba89743f38c2923d22ed", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1218, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-10", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 239823.56, + "total_prompt_tokens": 6391, + "total_retrieved_plane1_tokens": 4846, + "total_tokens": 9463, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1306, + "response_sha256": "2c9a984b71693ba519b23804f4a2c13dddf9b8216907ccc99b8ba0b4b1ed4e16", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2057, + "response_sha256": "cf7e2c9a54fb1cbff14db9b1a16d677ac2e0b03e9829b76d34a43d30dd286190", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1576, + "response_sha256": "f92c6cfb1be84e5d5b876a4bf7a5d888e148e959fa9e2991b2a20f44d735262e", + "retrieved_chunk_ids": [ + "sec12_69", + "sec12_72", + "sec5_20", + "sec6_30" + ], + "retrieved_plane1_tokens": 1203, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1452, + "response_sha256": "8e3e03b0e367150326bb3326057cc19fa932603a4726fd8f0cbfc07526fc15ef", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1218, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-11", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 239955.37, + "total_prompt_tokens": 6298, + "total_retrieved_plane1_tokens": 4841, + "total_tokens": 9370, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1346, + "response_sha256": "e8dbc341701f1a2e53bd986f40a0067467f8020ff69dc74205f4d5a194ae841b", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2058, + "response_sha256": "8fa1a7c23f92a34a42589b3f9840fbdac3f6a89c82ec3266e5517dffb92c670b", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1463, + "response_sha256": "0c1c87f502a740c271346e02c00d8300e98fde5e2caebba0c2991493729dbec9", + "retrieved_chunk_ids": [ + "sec12_69", + "sec5_15", + "sec6_26", + "sec6_31" + ], + "retrieved_plane1_tokens": 1185, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1431, + "response_sha256": "efe59e7745572e7743694b14da13491ad58c0cab92a023e561aae490429e4454", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1218, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-12", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 239992.84, + "total_prompt_tokens": 6587, + "total_retrieved_plane1_tokens": 4858, + "total_tokens": 9659, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1292, + "response_sha256": "6ad1ab024371a621aba69de9b78ca858f14c5fc443f28b8ad3c625bce2245a49", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1995, + "response_sha256": "2cca609b593d3bc953e22fc8a6cf64144c974f804f739c0d96980dcc89ee4fbb", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1875, + "response_sha256": "0a706ff687d9fff6c70a724451108222dd2a5ed86cfdbb6637eee7915d58cf56", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1425, + "response_sha256": "ebab498fa870c643b7c20034f52ea37ef958f77ab4f143dd9ce4be947ed18e41", + "retrieved_chunk_ids": [ + "sec12_69", + "sec12_72", + "sec5_15", + "sec6_26", + "sec6_30" + ], + "retrieved_plane1_tokens": 1201, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-13", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 240027.77, + "total_prompt_tokens": 6854, + "total_retrieved_plane1_tokens": 4840, + "total_tokens": 9926, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1439, + "response_sha256": "1d9cf69421bb54abc8ae4c32f99d35f284822d0a9186588cb8ea9cd4f766a48a", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2195, + "response_sha256": "04c1d02cba9d0fbc843b954071c6c1ddf0eaeb0ac88ed89ad3b5ebcb764ff455", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1624, + "response_sha256": "bc3e60e376f72cb1aefeed9227710cbc019685dacff13f4660b47e81839dc580", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1596, + "response_sha256": "c03db9cf6cb6c5dad114bfa5e02c1e3b5acaac59d19571f4c38ecbd1aea2b763", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_72", + "sec12_74", + "sec6_26", + "sec6_30" + ], + "retrieved_plane1_tokens": 1211, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-14", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 240074.58, + "total_prompt_tokens": 7887, + "total_retrieved_plane1_tokens": 4863, + "total_tokens": 10959, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1410, + "response_sha256": "08eb5c235082651b0fb437475957cab85e34bbbe556e00d4846b2103cc222db5", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2292, + "response_sha256": "40dd8ac445adb6caed455c8de64989af87bcbf2755ae9225f6e440704bf19d0f", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2079, + "response_sha256": "59e197f56aeb4c2c03ffd696ca3cd2386ffc5f749e1c2f6aabc75193d3dee9e8", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2106, + "response_sha256": "65ced1f8fff364f0daaabd428f7cec221d9d35b8102d0060b7d2b888b28dac30", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-15", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 240139.66, + "total_prompt_tokens": 7897, + "total_retrieved_plane1_tokens": 4861, + "total_tokens": 10969, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1487, + "response_sha256": "3b0868a1e079e855ab31cdf01a31fb989c41d637f1bc9c46c9b7eabaacb25da9", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2161, + "response_sha256": "eb8e825c7fc630e9642c1310aad37e79b830e003e8942dc90233423b59e79375", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2116, + "response_sha256": "68e71d49a95322e72f72a7ad5bf8136a4c76aea73441cfbc6e5d40249de7732e", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2133, + "response_sha256": "913b7febb0c6249ab4c092873fa282c89dde641bb6c49f1bc1c10d30868cf04f", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-16", + "inference_certified_at_turn": 4, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 240191.45, + "total_prompt_tokens": 7024, + "total_retrieved_plane1_tokens": 4849, + "total_tokens": 10096, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1323, + "response_sha256": "dd71ad3f0ce70470a2196128966d18b382ccf540375a814e4d279e4fb98b5a88", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec6_25", + "sec6_26", + "sec6_30" + ], + "retrieved_plane1_tokens": 1202, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2078, + "response_sha256": "e3f24bf89752d132a1d654c6c9fa3866bce8250251bb386735b044ab63204ccf", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2065, + "response_sha256": "080fecaa4705dac7243fc7d3a85637922f32fecc8c956f5146a4bb2862a002a7", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1558, + "response_sha256": "5dee0e61caa364b1d451898b271032201ed6a9921877fc54f03f08aadbd036d7", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_69", + "sec12_72", + "sec5_18", + "sec5_20" + ], + "retrieved_plane1_tokens": 1209, + "sealed_error_categories": [ + "valid_case_0_rejected", + "valid_case_1_rejected" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-17", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246051.23, + "total_prompt_tokens": 7467, + "total_retrieved_plane1_tokens": 4820, + "total_tokens": 10539, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1393, + "response_sha256": "b2d05da279a98f8497a4975556ca417289013d4659fec4a0cad80ab64b7d038f", + "retrieved_chunk_ids": [ + "sec5_16", + "sec5_20" + ], + "retrieved_plane1_tokens": 1208, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2029, + "response_sha256": "b1f7249222f284b909222226ab209783624b62d35e0eb65337a157dfd7b37e4c", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2029, + "response_sha256": "90a8ac4d6bfdb87c25a360fffadcca8c6bb65fe4b24c77728d3d0206c1d5c3c4", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2016, + "response_sha256": "90a8ac4d6bfdb87c25a360fffadcca8c6bb65fe4b24c77728d3d0206c1d5c3c4", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-18", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246101.53, + "total_prompt_tokens": 6269, + "total_retrieved_plane1_tokens": 4833, + "total_tokens": 9341, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1253, + "response_sha256": "65caa028c680a40bd2e47a85ded6f3e8d3f2153622d4a99c9f23974bb6809fde", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2008, + "response_sha256": "269be05b0f05be9cbdafe206dac6ee082d1668409e5756c3eaa59ad3334797d8", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1524, + "response_sha256": "e62e572b2b17445df808cc9844ae934778da27f2fe468a031631a99703205377", + "retrieved_chunk_ids": [ + "sec12_69", + "sec12_72", + "sec5_20", + "sec6_30" + ], + "retrieved_plane1_tokens": 1203, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1484, + "response_sha256": "a03b055ebdb8f8e9db572ab3ac223142d8b16337d7431a96efbaf40c5ada9128", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_68", + "sec12_74", + "sec6_27" + ], + "retrieved_plane1_tokens": 1205, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-19", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246205.89, + "total_prompt_tokens": 7511, + "total_retrieved_plane1_tokens": 4861, + "total_tokens": 10583, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1382, + "response_sha256": "12919162ed8f94bbfeed8b807d42f94733b5c12b856838dede15b912f26b005d", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2091, + "response_sha256": "b7ed828cf21091edc49c31cf1bff24257274cdbb23db9d38970b3773840062b1", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2026, + "response_sha256": "315c68f2d99dc1aa5d233ba7636ad084b9e730897292b8bb8a03b7406f7e7291", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2012, + "response_sha256": "41c170a0effecb8d928b8f2fdbf1a5f75eacf45cffffc51a04b3b72d71610b47", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-20", + "inference_certified_at_turn": 4, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246247.09, + "total_prompt_tokens": 5954, + "total_retrieved_plane1_tokens": 4849, + "total_tokens": 9026, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1226, + "response_sha256": "32abab45cf5aa1fd3bea8857b4bb2a16c47f700d895abd0e3771705a3c0ae7d5", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1218, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1802, + "response_sha256": "d7daeebc793749d340bbdae9216dfbffeede4cf04832968389d95cbdceef36ad", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1218, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1459, + "response_sha256": "9c9ac742e844b55ac63b887393bc945307a346be1ca6d217db8716ac930ba153", + "retrieved_chunk_ids": [ + "sec12_65", + "sec5_15", + "sec5_20" + ], + "retrieved_plane1_tokens": 1209, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1467, + "response_sha256": "fff44efaefb5721dea407406858bae41875737570d0372de158cb7f224c6dc4c", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec12_74", + "sec5_18", + "sec6_26", + "sec6_30" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "valid_case_0_rejected", + "valid_case_1_rejected" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-21", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246204.09, + "total_prompt_tokens": 7228, + "total_retrieved_plane1_tokens": 4795, + "total_tokens": 10300, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1398, + "response_sha256": "64b124be9792db4b78fd9222eadc8a52b167a99f5b5b29259faf16fef68b33d3", + "retrieved_chunk_ids": [ + "sec11_61", + "sec12_77", + "sec5_15", + "sec5_16", + "sec6_25" + ], + "retrieved_plane1_tokens": 1196, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2090, + "response_sha256": "5785e6e33b02f209b138680b2ad165cf6d83fd50f1d3a90e07a6b97f5ff795b3", + "retrieved_chunk_ids": [ + "sec11_61", + "sec12_77", + "sec5_15", + "sec5_16", + "sec6_25" + ], + "retrieved_plane1_tokens": 1196, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2107, + "response_sha256": "2da227f4a0fb7092396e060802a5b7daa0a4b70feb8d092c5fc4b9fc641f158a", + "retrieved_chunk_ids": [ + "sec11_61", + "sec12_77", + "sec5_15", + "sec5_16", + "sec6_25" + ], + "retrieved_plane1_tokens": 1196, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1633, + "response_sha256": "e37d1a75cd719071ed90e7b166f9a3904245c097d0a1135b13c2ccf1f113e4f6", + "retrieved_chunk_ids": [ + "sec12_77", + "sec6_27" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-22", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246312.12, + "total_prompt_tokens": 6364, + "total_retrieved_plane1_tokens": 4837, + "total_tokens": 9436, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1364, + "response_sha256": "49b477e330684f865fd2c00a9d0ef60fc768572fd5238fc0b7ea505d92f1fe6a", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1976, + "response_sha256": "dee0d1ddbf1448ecee9a86bbf04e0ab17540c1b72b8e0859b4898d6792b43243", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1486, + "response_sha256": "db00012cb19c8760a961cd96d6a8237dc99185a042a41a729860a504d53df9ae", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_69", + "sec12_72", + "sec6_26", + "sec6_30" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1538, + "response_sha256": "5c1c4c028ea4fde52fe6dc583f0a1d1931e2d7d68334b9e4e10f4d6c24a560c3", + "retrieved_chunk_ids": [ + "sec12_77", + "sec6_27" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-23", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246298.76, + "total_prompt_tokens": 6133, + "total_retrieved_plane1_tokens": 4851, + "total_tokens": 9205, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1291, + "response_sha256": "88f975d3d40aaf954fb8197a4e45f7898297db4f2b5f0849543cda54363d2860", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1887, + "response_sha256": "4e373df40b428f8eb705560c947392a82537eb3e012bd793c80e734bd101cabf", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1518, + "response_sha256": "a6accefd346bf0f1ef4f3d5e1e5ff3d4ffa9423c55df4c193ff060123fd8b267", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1437, + "response_sha256": "5590406ed96e40c1492896efb780d8cbdf381bd85a19df0684925192a1bf3e4f", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1218, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-24", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246345.56, + "total_prompt_tokens": 6690, + "total_retrieved_plane1_tokens": 4833, + "total_tokens": 9762, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1307, + "response_sha256": "9fad7795f0841a71cb82f2a379d99e449b875947c82ef9a3e17ccfcc2398e3d1", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1942, + "response_sha256": "f97652f18bab599feef267ef0b56e9f495d4a22bb08830ced4b51cf2f26cbefe", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1491, + "response_sha256": "38afbad18687d1dee188ab2cf661437ca2755294a4da6c4dc404587e02610b9b", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1950, + "response_sha256": "84ff82e288386660af1d03d5e53791d30a84294e5e3d1c0db2dcece5a3b53d4a", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-01", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 598, + "total_elapsed_ms": 56149.44, + "total_prompt_tokens": 1103, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1701, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 598, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1103, + "response_sha256": "11badb2542511fcd6ada8f41aa9e05d088ee39ab173b1715f4a601f99ef7627b", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_required_boundary_fact" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-02", + "inference_certified_at_turn": 2, + "oracle_calls": 2, + "stagnation_events": 0, + "total_completion_tokens": 701, + "total_elapsed_ms": 62767.81, + "total_prompt_tokens": 2518, + "total_retrieved_plane1_tokens": 2452, + "total_tokens": 3219, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 338, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1046, + "response_sha256": "db54ff6139883fad24c4fb0642d684f730a96e48c1a2c758f78aff256f91f541", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "missing_required_boundary_fact" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 363, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1472, + "response_sha256": "1b9885d83efb955faeb5681e9130e6400dbb51c85f70a61bf5ac0b0566f38d90", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words", + "missing_required_boundary_fact" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-03", + "inference_certified_at_turn": 2, + "oracle_calls": 2, + "stagnation_events": 0, + "total_completion_tokens": 835, + "total_elapsed_ms": 70259.91, + "total_prompt_tokens": 2547, + "total_retrieved_plane1_tokens": 2408, + "total_tokens": 3382, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 334, + "inference_diagnostic_rules": [ + "INCOMPLETE_TRUST_BOUNDARY" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1073, + "response_sha256": "6468b9d7be9ba3a8d41056adbfb4de690521c9cf2d59af1fa43b43d3b51f870b", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "missing_required_boundary_fact" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 501, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1474, + "response_sha256": "f0bf6d609114fd033afb5aab3b2c86a73f4ac8b2931720617164005a75951cfe", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "missing_required_boundary_fact" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": true, + "deployed_pass_at_3": true, + "deployed_pass_at_4": true, + "family": "C", + "id": "HOLD-C-04", + "inference_certified_at_turn": 2, + "oracle_calls": 2, + "stagnation_events": 0, + "total_completion_tokens": 717, + "total_elapsed_ms": 63753.84, + "total_prompt_tokens": 2494, + "total_retrieved_plane1_tokens": 2452, + "total_tokens": 3211, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": true, + "trajectory_any_pass_at_3": true, + "trajectory_any_pass_at_4": true, + "turns": [ + { + "completion_tokens": 309, + "inference_diagnostic_rules": [ + "INCOMPLETE_TRUST_BOUNDARY" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1059, + "response_sha256": "70abb940cedd9ecafbfab4ad4591ef6e485ee2a07bfb42987afa38cb219d328e", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 408, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1435, + "response_sha256": "f274361e6a50c46b2f81590ce8a9a90ab9195684f43e9529e17358b43bce7142", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [], + "sealed_holdout_passed": true, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-05", + "inference_certified_at_turn": 2, + "oracle_calls": 2, + "stagnation_events": 0, + "total_completion_tokens": 583, + "total_elapsed_ms": 55446.4, + "total_prompt_tokens": 2410, + "total_retrieved_plane1_tokens": 2408, + "total_tokens": 2993, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 239, + "inference_diagnostic_rules": [ + "INCOMPLETE_TRUST_BOUNDARY" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1052, + "response_sha256": "8765a96b9c8064aa17e05670cca04429c539c708a4fd1ded0eb3dae8825a9e24", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "missing_expected_tier_label" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 344, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1358, + "response_sha256": "ec01f783c8f4d1ecec8314d1bfdc64477ad6700186c8c1af7a235938d0358fc6", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "missing_expected_tier_label" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-06", + "inference_certified_at_turn": 2, + "oracle_calls": 2, + "stagnation_events": 0, + "total_completion_tokens": 1286, + "total_elapsed_ms": 81900.1, + "total_prompt_tokens": 2646, + "total_retrieved_plane1_tokens": 2452, + "total_tokens": 3932, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 540, + "inference_diagnostic_rules": [ + "INCOMPLETE_TRUST_BOUNDARY" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1053, + "response_sha256": "958e2645d49d05e73548fa2467319790b37fb0c0bcc548e3e637deb6f52ef8c0", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 746, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1593, + "response_sha256": "5cd7bacdcaa0ef570aae423e62140597a73bf523c76095d9d51ba308dbe27048", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words", + "missing_expected_tier_label" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-07", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 344, + "total_elapsed_ms": 36346.82, + "total_prompt_tokens": 1061, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1405, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 344, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1061, + "response_sha256": "343fd895a2c522bd0abd4d237663018bf9fa48fe53cf71315b0382eb2af332e8", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words", + "missing_required_boundary_fact" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-08", + "inference_certified_at_turn": 2, + "oracle_calls": 2, + "stagnation_events": 0, + "total_completion_tokens": 1006, + "total_elapsed_ms": 76972.5, + "total_prompt_tokens": 2617, + "total_retrieved_plane1_tokens": 2408, + "total_tokens": 3623, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 478, + "inference_diagnostic_rules": [ + "INCOMPLETE_TRUST_BOUNDARY" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1036, + "response_sha256": "f79226640e37ad255939a698658a5cfa4fe8ca497bc287fd1b74d527820d45c8", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 528, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1581, + "response_sha256": "591b05ad238b3f33bce773a016a04c6e542ca7f460bd3056bf7db57680aac453", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-09", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2776, + "total_elapsed_ms": 227334.41, + "total_prompt_tokens": 7399, + "total_retrieved_plane1_tokens": 4880, + "total_tokens": 10175, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 472, + "inference_diagnostic_rules": [ + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1261, + "response_sha256": "8ea6698deaf7d3c0e3aa1aeaf2c16b89bb24c9af81407fa6cb3bad3def144010", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_expected_tier_label", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "INVALID_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2045, + "response_sha256": "9ac9be833ce9df3519382dab1da5f79b70ce2e2b32c9ee76a39364846eb6257b", + "retrieved_chunk_ids": [ + "sec4_09", + "sec5_15", + "sec8_39", + "sec8_41" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_trust_words", + "wrong_declared_tier", + "mde_json_audit", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST", + "INVALID_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2050, + "response_sha256": "022a592e43c747dc5591e14499d728485c17314bf0cdb62e489ba6afe5e5213c", + "retrieved_chunk_ids": [ + "sec7_34", + "sec8_36", + "sec8_39", + "sec9_48" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_expected_tier_label", + "mde_json_audit", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST", + "INVALID_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2043, + "response_sha256": "518d0952e416b2c9d2a66710b7345b761c775f399cb8b7acc8597a865c84a340", + "retrieved_chunk_ids": [ + "sec12_69", + "sec4_09", + "sec7_32", + "sec7_34", + "sec8_39" + ], + "retrieved_plane1_tokens": 1213, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_expected_tier_label", + "mde_json_audit", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-10", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 2298, + "total_elapsed_ms": 198831.44, + "total_prompt_tokens": 6762, + "total_retrieved_plane1_tokens": 4876, + "total_tokens": 9060, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 342, + "inference_diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION", + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1266, + "response_sha256": "b2d0d3af77be449db1cf80b03d62aa307bf07023907896319b3aefaa6fe316cc", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_required_boundary_fact", + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 420, + "inference_diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1689, + "response_sha256": "a7fecebf85622b7128a2cc5762908805ef300a460463853726470ee8aeee9a1d", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1917, + "response_sha256": "e6f48dd338fb93aa3e9437e0a795377fc50db236747bd27c89a4f1ec09968432", + "retrieved_chunk_ids": [ + "sec7_32", + "sec8_36", + "sec8_37", + "sec9_44" + ], + "retrieved_plane1_tokens": 1203, + "sealed_error_categories": [ + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1890, + "response_sha256": "3fe31e94ee040b13166c6a9e590e960ecba250824691c7a3113e6e1543d1a578", + "retrieved_chunk_ids": [ + "sec7_34", + "sec8_36", + "sec8_39", + "sec9_48" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_trust_words", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-11", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 2267, + "total_elapsed_ms": 197267.83, + "total_prompt_tokens": 6938, + "total_retrieved_plane1_tokens": 4884, + "total_tokens": 9205, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 326, + "inference_diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION", + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1276, + "response_sha256": "9ea3e0c98d53fcc441be2a1a51a64c886feeca6fc8c71bb288da7dd178858781", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_expected_tier_label", + "missing_required_boundary_fact", + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 405, + "inference_diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1683, + "response_sha256": "db41e4fbb525d2cdd8eaf45484ed2737290284a2002d4d6cca3d2e0ffc54783a", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_expected_tier_label", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1924, + "response_sha256": "cb07c46b43d965688ee0d58703c567c70ac525379054feca89a47e1db6cede04", + "retrieved_chunk_ids": [ + "sec4_09", + "sec7_32", + "sec8_36", + "sec8_37", + "sec9_48" + ], + "retrieved_plane1_tokens": 1212, + "sealed_error_categories": [ + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2055, + "response_sha256": "87c540901ca9c26f2d236ff1e20469d314b1f41475aa51dd7aa7936c76c6edba", + "retrieved_chunk_ids": [ + "sec4_09", + "sec5_15", + "sec8_39", + "sec8_41" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_or_empty_section", + "mde_json_audit", + "mde_json_audit", + "mde_json_audit", + "mde_json_audit", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-12", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 2219, + "total_elapsed_ms": 192076.95, + "total_prompt_tokens": 6662, + "total_retrieved_plane1_tokens": 4876, + "total_tokens": 8881, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 302, + "inference_diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION", + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1263, + "response_sha256": "75eca15a91b5ec062ca19f051374ac66706c99187bec6d85d42fbcc811e66585", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_required_boundary_fact", + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 381, + "inference_diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1646, + "response_sha256": "37171e4a3f4d39e2fcdc2b1011209a8deb9b33dccc119cf78c441092cd12de3c", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1868, + "response_sha256": "68df74b1f508460a027287afb17ee88779eae3086fb06080db64bf6f9265c81f", + "retrieved_chunk_ids": [ + "sec7_32", + "sec8_36", + "sec8_37", + "sec9_44" + ], + "retrieved_plane1_tokens": 1203, + "sealed_error_categories": [ + "missing_required_boundary_fact", + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1885, + "response_sha256": "49f0c8bfb59d9c32109763164d6f7980607a7683514be5e4dac82065136ffa78", + "retrieved_chunk_ids": [ + "sec7_34", + "sec8_36", + "sec8_39", + "sec9_48" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_required_boundary_fact", + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-13", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3023, + "total_elapsed_ms": 235712.28, + "total_prompt_tokens": 7632, + "total_retrieved_plane1_tokens": 4849, + "total_tokens": 10655, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1356, + "response_sha256": "fee7c197d76e1da662d3fa0d7607f36ce3874a760c62ab75729152c8dc930503", + "retrieved_chunk_ids": [ + "sec7_34", + "sec8_36", + "sec8_39", + "sec9_48" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_expected_tier_label", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 751, + "inference_diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2065, + "response_sha256": "9162da6faf580ed599d990d6475ef62deb765475bc455679e3c89cf619fde02f", + "retrieved_chunk_ids": [ + "sec4_09", + "sec5_15", + "sec8_39", + "sec8_41" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_expected_tier_label", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 736, + "inference_diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2074, + "response_sha256": "5a216f0ac96be1cadd79616d3e2db68507dce8ae48ada8225bee1b2eba59909d", + "retrieved_chunk_ids": [ + "sec12_65", + "sec8_39", + "sec8_41" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "missing_expected_tier_label", + "missing_required_boundary_fact", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2137, + "response_sha256": "6d057b01a5f9d0f6cf13574e82bf28b6919bc27d4b3962e48c49a1493007b13c", + "retrieved_chunk_ids": [ + "sec12_65", + "sec8_39", + "sec8_41" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "missing_expected_tier_label", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-14", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2983, + "total_elapsed_ms": 233986.07, + "total_prompt_tokens": 7562, + "total_retrieved_plane1_tokens": 4860, + "total_tokens": 10545, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 690, + "inference_diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1280, + "response_sha256": "9916cd6993fb4e55074515d50bf01215a1c06a28dc6a518a455aca56cdcb0ea5", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_expected_tier_label", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 757, + "inference_diagnostic_rules": [ + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2181, + "response_sha256": "51293871edc351620920d7c4bf1905caee903c8db6474acb06a8556ab0abc909", + "retrieved_chunk_ids": [ + "sec7_32", + "sec8_36", + "sec8_37", + "sec9_44" + ], + "retrieved_plane1_tokens": 1203, + "sealed_error_categories": [ + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2107, + "response_sha256": "2d44eb4e21004113436fee6df9d50ac7d77c7346d5e18c91fb191babfa71bd15", + "retrieved_chunk_ids": [ + "sec10_56", + "sec8_39", + "sec8_41" + ], + "retrieved_plane1_tokens": 1210, + "sealed_error_categories": [ + "missing_expected_tier_label", + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1994, + "response_sha256": "a78f137276529f8dc1fe8abc0ce7ba75dbd7e653c41670810f3dbc36454edb44", + "retrieved_chunk_ids": [ + "sec7_34", + "sec8_36", + "sec8_39", + "sec9_48" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_expected_tier_label", + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-15", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 236832.53, + "total_prompt_tokens": 7141, + "total_retrieved_plane1_tokens": 4880, + "total_tokens": 10213, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1344, + "response_sha256": "0a26efb9768142e363ce76a9ea38af3663ce23df78d9f8451c40a55c852845ba", + "retrieved_chunk_ids": [ + "sec7_34", + "sec8_36", + "sec8_39", + "sec9_48" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "missing_expected_tier_label", + "wrong_verdict_value" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1870, + "response_sha256": "93d226dacb8bdd7ebbf16b2162c6b6897d6adea31e35679b497b8094d56718bc", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words", + "missing_expected_tier_label", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2029, + "response_sha256": "bebdaa67c34dfece5a1a9c671d59269f12d44b84487b3a8ed5645def87642959", + "retrieved_chunk_ids": [ + "sec4_09", + "sec5_15", + "sec8_39", + "sec8_41" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_expected_tier_label", + "wrong_verdict_value" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1898, + "response_sha256": "e9792e9461f9d553a22be06d5b1cabc193b5df5595b42251db24a96c97090124", + "retrieved_chunk_ids": [ + "sec12_69", + "sec4_09", + "sec7_32", + "sec7_34", + "sec8_39" + ], + "retrieved_plane1_tokens": 1213, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_expected_tier_label", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-16", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 236865.57, + "total_prompt_tokens": 8221, + "total_retrieved_plane1_tokens": 4859, + "total_tokens": 11293, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1458, + "response_sha256": "b295724db15416b11763390d2c08cbe0561e5f1114e8aaaaa6c24828dd918116", + "retrieved_chunk_ids": [ + "sec7_34", + "sec8_36", + "sec8_39", + "sec9_48" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_trust_words", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2210, + "response_sha256": "6a66e4aa9ab093e092ffe080148816c81f9c95739455cb5122c82917def07b37", + "retrieved_chunk_ids": [ + "sec7_32", + "sec8_36", + "sec8_37", + "sec9_44" + ], + "retrieved_plane1_tokens": 1203, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_expected_tier_label", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2324, + "response_sha256": "1d55bc4c8b485b2ed485c778d89c46056629655c2762fc46d3041a5322f5cff3", + "retrieved_chunk_ids": [ + "sec8_39", + "sec8_41", + "sec9_42" + ], + "retrieved_plane1_tokens": 1224, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_expected_tier_label" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST", + "PROMPT_EVIDENCE_CONTRADICTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2229, + "response_sha256": "f4f9f68961802d494dd5eace284eb0b8a0a5cd309695f82d4fbdb13a939a24bb", + "retrieved_chunk_ids": [ + "sec4_09", + "sec6_25", + "sec7_32", + "sec7_34", + "sec8_39" + ], + "retrieved_plane1_tokens": 1211, + "sealed_error_categories": [ + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-17", + "inference_certified_at_turn": 2, + "oracle_calls": 2, + "stagnation_events": 0, + "total_completion_tokens": 808, + "total_elapsed_ms": 62245.09, + "total_prompt_tokens": 2840, + "total_retrieved_plane1_tokens": 2452, + "total_tokens": 3648, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 394, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1179, + "response_sha256": "012fbcdc622294d7cba5b4860417ae76b0b9766a0dc59e3406c377dd71a46777", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "missing_required_boundary_fact", + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 414, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1661, + "response_sha256": "f9741ab26958faf2fff98ef7b176061b6296e2cce53bac1849b119785b003a6a", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_required_boundary_fact", + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-18", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 604, + "total_elapsed_ms": 51012.63, + "total_prompt_tokens": 1234, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1838, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 604, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1234, + "response_sha256": "36a13301afe7bdcd25085552d1b1af82048631231322d4e8dd12b4cd31acc75b", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "f0_gate", + "f0_gate" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-19", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 375, + "total_elapsed_ms": 31333.78, + "total_prompt_tokens": 1182, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1557, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 375, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1182, + "response_sha256": "9c085f54ae90dbfe8aa88f2569e1ded4b047139768946edeb0cb1f41893b1932", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_required_boundary_fact", + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-20", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 471, + "total_elapsed_ms": 39327.72, + "total_prompt_tokens": 1228, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1699, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 471, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1228, + "response_sha256": "2a82c643f4733dada703b76b4ffd5d2628dedb73ca2ed828bd5a1089144c3e28", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words", + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-21", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 442, + "total_elapsed_ms": 37415.23, + "total_prompt_tokens": 1190, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1632, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 442, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1190, + "response_sha256": "7d1fc080c426b507869d5f3662d88a8b0a1facf21bd1d1e318da7c005658fcc8", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_expected_tier_label", + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-22", + "inference_certified_at_turn": 2, + "oracle_calls": 2, + "stagnation_events": 0, + "total_completion_tokens": 1295, + "total_elapsed_ms": 77349.27, + "total_prompt_tokens": 3097, + "total_retrieved_plane1_tokens": 2452, + "total_tokens": 4392, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 527, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1241, + "response_sha256": "22dee7a325406b3f2b47c3ff558b23f32150b6dcb5f096e28754216f917c9532", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1856, + "response_sha256": "ad48c3276fadb160d37c3adbb895c097907eb0dc129996b96520b86448e5b85c", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-23", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 457, + "total_elapsed_ms": 38491.9, + "total_prompt_tokens": 1176, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1633, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 457, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1176, + "response_sha256": "1633c33af704f2505e1b0bd2ee910f3ee3ee3c21528e47d72dcc3ea5522ff8a9", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words", + "missing_expected_tier_label", + "missing_required_boundary_fact", + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-24", + "inference_certified_at_turn": 2, + "oracle_calls": 2, + "stagnation_events": 0, + "total_completion_tokens": 1146, + "total_elapsed_ms": 74769.69, + "total_prompt_tokens": 3093, + "total_retrieved_plane1_tokens": 2452, + "total_tokens": 4239, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 539, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1233, + "response_sha256": "320336fc266f65967705c3ea809c763b2d3d8098e48ad7dd5af97bb2f8b57e12", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 607, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1860, + "response_sha256": "ada1e93e3fb61b992ebfe76951ee9c85b1acf98e07a2d9f811685ff3b76f657b", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-01", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1166, + "total_elapsed_ms": 121306.53, + "total_prompt_tokens": 6175, + "total_retrieved_plane1_tokens": 4857, + "total_tokens": 7341, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 241, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1204, + "response_sha256": "1b7e3c1cfa879129f7748a08e23f3cdf6847eae4e5e1a9b48dbc50978daae0a9", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 305, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1644, + "response_sha256": "b672057fe477ed9e913899fb3ac487d91ba30f18057e360d05bc02cc48db7417", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 305, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1701, + "response_sha256": "b672057fe477ed9e913899fb3ac487d91ba30f18057e360d05bc02cc48db7417", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 315, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1626, + "response_sha256": "e08747bfaa6f5f74530b8d0135914273525ca1c026f0d1aa8c9c1614d8039107", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-02", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 1704, + "total_elapsed_ms": 147346.75, + "total_prompt_tokens": 6666, + "total_retrieved_plane1_tokens": 4857, + "total_tokens": 8370, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 296, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1196, + "response_sha256": "653acfe50502c38749aac8396815e8513a5088a7f1e8d1e65bd5a8477bcef3d2", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 337, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1761, + "response_sha256": "02e42f195d484ad4a92e01767c2d17a4d683321a411da7e5f1e8075f49b18a3b", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1745, + "response_sha256": "06d76544156f71a2b4395a08db4e1347e5521dbed27c98ecb93a87694d734c17", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 303, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1964, + "response_sha256": "e1a79e49d9c21800b7bbd11662e9aafe9fcc5487b0d879e2262305d9676bcb92", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-03", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 1541, + "total_elapsed_ms": 144324.65, + "total_prompt_tokens": 6738, + "total_retrieved_plane1_tokens": 4857, + "total_tokens": 8279, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 334, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NO_THEOREM_DECLARED", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1199, + "response_sha256": "773b4b3e4d5c0fc20f3fb2d11ca1c5ed61c16eff94dc49760300520e142a95c0", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 392, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1867, + "response_sha256": "dc1b104b423400eb26c20f727364f1fde361767e9491851da068da3d2048e766", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 413, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1836, + "response_sha256": "2f1a60b986f9e4561fa339ad60e9ea7a5d235433e2b4586e222768128931e4cd", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 402, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1836, + "response_sha256": "82f76be834294e74fe8d2e7ac986ddeffc7e407bd799a9232c5b62412bab9f6d", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-04", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1278, + "total_elapsed_ms": 128921.09, + "total_prompt_tokens": 6323, + "total_retrieved_plane1_tokens": 4865, + "total_tokens": 7601, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 273, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1211, + "response_sha256": "f74dc741ca06c11d7edeeb93f1a4050be05b5682267912025dcd834eec9c6800", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 337, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1682, + "response_sha256": "bfe729e2c2c3940dd76c14edafde8818c3324af5504b1430a2676b3920cb5bc7", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 340, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1759, + "response_sha256": "77c326cb90f680aa28c83bd6415b76ec1dd52533cc08913b10eedbc9301005f5", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 328, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1671, + "response_sha256": "85f6ab6042f67c45c1a0b80e49eb887187a49890b1947983fbff71c346023d08", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-05", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1504, + "total_elapsed_ms": 142947.96, + "total_prompt_tokens": 6683, + "total_retrieved_plane1_tokens": 4849, + "total_tokens": 8187, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 323, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1210, + "response_sha256": "9f25a584f706200f09e6e4e21b5ffdc392181cb8c867dcf06dc5c60d781580db", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 387, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1836, + "response_sha256": "90569bd9dd85b2efc9781b027298fe65bfdb3d6427033f5afd6a46b0b0282084", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 397, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_FORBIDDEN_META_IMPORT", + "EMBEDDED_LEAN_FORBIDDEN_META_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1822, + "response_sha256": "65ad437e4357c6059ee30c56e0d64ccfe16c7311ed6d4b090ac23537df92f6ba", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 397, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_FORBIDDEN_META_IMPORT", + "EMBEDDED_LEAN_FORBIDDEN_META_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1815, + "response_sha256": "65ad437e4357c6059ee30c56e0d64ccfe16c7311ed6d4b090ac23537df92f6ba", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-06", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1515, + "total_elapsed_ms": 144099.01, + "total_prompt_tokens": 6673, + "total_retrieved_plane1_tokens": 4866, + "total_tokens": 8188, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 372, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1208, + "response_sha256": "81c43a517d56e719bcaf06a92c294ef025ee9eb236448382d62af417c74eb299", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "lean4_auditor", + "missing_trust_words", + "mde_guard", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 423, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1914, + "response_sha256": "bfea33094414c43145cd9815560fe503bf5e544f0322aeba03cd73635d410116", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words", + "mde_guard", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 423, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1864, + "response_sha256": "bfea33094414c43145cd9815560fe503bf5e544f0322aeba03cd73635d410116", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec12_66", + "sec12_69" + ], + "retrieved_plane1_tokens": 1216, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words", + "mde_guard", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 297, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1687, + "response_sha256": "40bbb9e55855ecc8f3f8c16c7df7b6490581142f0a84eb2409ac31a6c4ee936a", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-07", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1472, + "total_elapsed_ms": 141538.6, + "total_prompt_tokens": 6410, + "total_retrieved_plane1_tokens": 4857, + "total_tokens": 7882, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 377, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1219, + "response_sha256": "958d0c3c6173cd469643a31cf495145f7ab4ad08819a0b239909153097cea169", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 435, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1837, + "response_sha256": "5feef144e181ba10473349ea858b8dd743ba0aa9f5afc0216e204103aa8f497c", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 334, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1676, + "response_sha256": "4f5f287bf1a5d19e8ae7edad39db3ec1e6cc7bf96714b169ceed8dfc7ef6a621", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 326, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1678, + "response_sha256": "864a1939f4ce4bb8c9d1d08ef6da61e7a62c9ed59c2c57448bf4c313977a4527", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-08", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1499, + "total_elapsed_ms": 142855.37, + "total_prompt_tokens": 6580, + "total_retrieved_plane1_tokens": 4849, + "total_tokens": 8079, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 301, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1200, + "response_sha256": "95a9b623557f348d55c7b5702862b5bdc663a4545bcf2f5e2fa262d0d79dd05a", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 404, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1702, + "response_sha256": "497d27196bb8b509e18a6b49a6645028ac646adc314eded6e840a15eb9620541", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 397, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1827, + "response_sha256": "5fcd0c2453ca72dd7465daa5c585dca441bb7e9373a82de5d5430edd0eace57f", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 397, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1851, + "response_sha256": "5fcd0c2453ca72dd7465daa5c585dca441bb7e9373a82de5d5430edd0eace57f", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-09", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1482, + "total_elapsed_ms": 149019.99, + "total_prompt_tokens": 6528, + "total_retrieved_plane1_tokens": 4857, + "total_tokens": 8010, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 291, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1213, + "response_sha256": "3e83d764f45ff9c8da48d28d06add730ad56ddbaf4b75e765ef6469e13b5a2b7", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 383, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1701, + "response_sha256": "150470f605e872f31486b893e237faa54eff2b0cebf9b58167c8842b835fa7bb", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 404, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1807, + "response_sha256": "10be2aa199bec1cec39f37941ca882d8fbf1060e8815f04c4768b9847036564a", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 404, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1807, + "response_sha256": "10be2aa199bec1cec39f37941ca882d8fbf1060e8815f04c4768b9847036564a", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-10", + "inference_certified_at_turn": 4, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1270, + "total_elapsed_ms": 139223.8, + "total_prompt_tokens": 6112, + "total_retrieved_plane1_tokens": 4857, + "total_tokens": 7382, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 297, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1197, + "response_sha256": "5d71162ab7c4aaaebb0bd2c69a24044dc8cf79828159a601169e53616dc591b8", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 349, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1694, + "response_sha256": "a580402b889a4acd4969054b626bf0b038bb69bf2d73b15a0ecc2525cc5b2960", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 349, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1668, + "response_sha256": "a580402b889a4acd4969054b626bf0b038bb69bf2d73b15a0ecc2525cc5b2960", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 275, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1553, + "response_sha256": "9babafe2ef41d79e5775832698afa17465e4f7b4018913fd8ff69a45430d8a99", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-11", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2083, + "total_elapsed_ms": 169717.32, + "total_prompt_tokens": 7298, + "total_retrieved_plane1_tokens": 4858, + "total_tokens": 9381, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 489, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1221, + "response_sha256": "b5a42f5d333d9c4d7692313d026590101dcc4394aa9c7963984953c2f3b5e9d8", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 552, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2045, + "response_sha256": "58ddf2fbcf6796ec2814e91d556c4e07fb231bc3b406b41fecc9dd68881b1448", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 521, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2030, + "response_sha256": "9c0a184bd84224afce68031838eb9895fbb23d57a7890e2ed7951897c9e141dc", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 521, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2002, + "response_sha256": "98c86910722cf75d216fb4d6c51ab60b75cf52a7807c53abdea5d91451dc8db1", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec12_66", + "sec12_69" + ], + "retrieved_plane1_tokens": 1216, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-12", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1131, + "total_elapsed_ms": 120667.59, + "total_prompt_tokens": 6143, + "total_retrieved_plane1_tokens": 4857, + "total_tokens": 7274, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 229, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1204, + "response_sha256": "b421743708383bdd4e1ed942685931a7c42b567a126e1a3a06c0403f90b8dee0", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 298, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1620, + "response_sha256": "66a22097701864bab0312430d2ab9b0ef48fc3f0c415574f1abfd459f3cf6e99", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 311, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1692, + "response_sha256": "d3f13d760b3277e96ed78f246588bfa8a1ccf68f97f23ea6819f6b96d46d2d24", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 293, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1627, + "response_sha256": "a1cac44f94020c2884601364617e6e45aa89aa7b0d65d977da6be22814b252f0", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-13", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1790, + "total_elapsed_ms": 163057.61, + "total_prompt_tokens": 6892, + "total_retrieved_plane1_tokens": 4849, + "total_tokens": 8682, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 408, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1211, + "response_sha256": "9af100ba55bc07940480a2c554ba3611d96806bed126ce4e52b35f977c1e47aa", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 460, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1951, + "response_sha256": "74e702989d4b7d2444b964852d212aa660dcdcf6a52a9e0094cc0d3deaf3bb51", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 461, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1892, + "response_sha256": "1147b2038ac66ccb7df5896fce8b0b2d3f76fcf698f7af75a747d0633cdb1813", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 461, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1838, + "response_sha256": "1147b2038ac66ccb7df5896fce8b0b2d3f76fcf698f7af75a747d0633cdb1813", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-14", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1401, + "total_elapsed_ms": 143358.61, + "total_prompt_tokens": 6582, + "total_retrieved_plane1_tokens": 4849, + "total_tokens": 7983, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 312, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1207, + "response_sha256": "d2a5118bc90d411025eaf5ce8f7ebd6b706ed9c6dad0ec5670dd9b5aebaf26ce", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 377, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1818, + "response_sha256": "64730faecee80e0e3afb2811f73da84839c7bbf6d7b317d5bb623584ddd39bc0", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 356, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1805, + "response_sha256": "1023959c33b1574e2d7b9478c2289e11854e32f11f541f2a9e0a152649a79754", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 356, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1752, + "response_sha256": "1023959c33b1574e2d7b9478c2289e11854e32f11f541f2a9e0a152649a79754", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-15", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 1126, + "total_elapsed_ms": 119890.94, + "total_prompt_tokens": 6188, + "total_retrieved_plane1_tokens": 4862, + "total_tokens": 7314, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 269, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1212, + "response_sha256": "0fdeccfacab59a6ba8dc279148b3cdb5752a774b7fc2bb87f636064c2f54ecf8", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 271, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1537, + "response_sha256": "87d6706c5a1b77cee2a631c4470cbe4c020c954dcc9a37943328c07c69c6f9fe", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 294, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1713, + "response_sha256": "bef8ee1b3105afa9ff451658665072520bdcd42b9fb13f14b0f9259f6fd163cf", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 292, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_FORBIDDEN_META_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1726, + "response_sha256": "8000750cc6bf2e8fd3886e0b26cd34eb6fd11ba069769dc0a476b4e9a03b0552", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-16", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 1404, + "total_elapsed_ms": 142946.41, + "total_prompt_tokens": 6529, + "total_retrieved_plane1_tokens": 4849, + "total_tokens": 7933, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 315, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1205, + "response_sha256": "3b13d45522f4f46fadcb89fa017f1971292c5216e27f87fbf2229ba88e7972c4", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 373, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1823, + "response_sha256": "1eaf6eb7a1ff5b93e66f264d99a7df618da8ad0130b37b5a691fb19d20df29fe", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 353, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_FORBIDDEN_META_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1803, + "response_sha256": "275a71ad2d0368ec90eab5b2acf926dc5700ab930cf7079205cf50151e551344", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 363, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1698, + "response_sha256": "585a80c3d926c059b5ee81216d1e93237f6040ad41a8ca61e6ae947488184fff", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-17", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 1396, + "total_elapsed_ms": 142668.5, + "total_prompt_tokens": 6436, + "total_retrieved_plane1_tokens": 4849, + "total_tokens": 7832, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 293, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1199, + "response_sha256": "871c1216085d827f733dc6def2cee01db47111475116e5ec3ff9e6be4536a272", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 351, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1761, + "response_sha256": "8eba1ac1bc6436e1eb4b4a0333a5c94d2f61e99c26a1bdcf05b9c98994f8ca03", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 375, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_FORBIDDEN_META_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1741, + "response_sha256": "28e2966aeeb92203b4aa112b74764dd5fa0265f25f42888b1afdb6ec855e454e", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 377, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1735, + "response_sha256": "5977704dba839c8d075f38284fd7ae143b6c44422365cfd49b83a0e6cd23e978", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-18", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1561, + "total_elapsed_ms": 152121.98, + "total_prompt_tokens": 6796, + "total_retrieved_plane1_tokens": 4858, + "total_tokens": 8357, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 319, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1216, + "response_sha256": "5d14e3595d565d9424ce6583e1d49ddfa66b38e429f282b75911b3d30ee224ca", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 326, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1776, + "response_sha256": "69115b410038bb90d49090c41f24f4523dc4a8dc9792a78c2bb03bf23208983e", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 590, + "inference_diagnostic_rules": [ + "PROSE_DUPES_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1762, + "response_sha256": "a8aaa61783c25d9c9cfcf717370b99a218525554c3ebf4163a8afe56adb825eb", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words", + "check_prose_failed", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 326, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2042, + "response_sha256": "69115b410038bb90d49090c41f24f4523dc4a8dc9792a78c2bb03bf23208983e", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec12_66", + "sec12_69" + ], + "retrieved_plane1_tokens": 1216, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-19", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1327, + "total_elapsed_ms": 136845.4, + "total_prompt_tokens": 6692, + "total_retrieved_plane1_tokens": 4865, + "total_tokens": 8019, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 256, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1219, + "response_sha256": "a316c21fbcf12775a0561637ed8112d69cf9df09b690ba9c735e40e38432fb41", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 478, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INVALID_CLAIM_TIER_VOCABULARY" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1773, + "response_sha256": "f21a9069d7fed85bb88a8fdce462d1be1d475a027a46ba2cb3bd4c289986ae21", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 296, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1944, + "response_sha256": "46c713162b81b140073d84eef7e8319ef2b965a9584b918f48ef171a900b1447", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 297, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1756, + "response_sha256": "7c9fb3a3e102b74a323d16165721fa025bad725bd3e472243cf59ec7cfb6033d", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-20", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1483, + "total_elapsed_ms": 147802.35, + "total_prompt_tokens": 6447, + "total_retrieved_plane1_tokens": 4849, + "total_tokens": 7930, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 374, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1198, + "response_sha256": "899ae16cba0623e4e9054515dd38981839172a7c084eb57d5b8eba546ed2905b", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 365, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1798, + "response_sha256": "565a0896de45b40388e8f95d58906ef88f3eff212f0a864372cd53ba3affd271", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 369, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1585, + "response_sha256": "f14c49421197d93cc4a09999a8a79298d0be09be7627d08d25d0fad2c659cf89", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 375, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1866, + "response_sha256": "df24bf011a90b4ad3a637bd512ddcbc9d33edf238a36eea40cfbcb26e89aa5e1", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-21", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1776, + "total_elapsed_ms": 157137.91, + "total_prompt_tokens": 7052, + "total_retrieved_plane1_tokens": 4866, + "total_tokens": 8828, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 417, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1222, + "response_sha256": "f01a936f094d27023a1b6023ec4e9c4efbe847bf155d54d8c662fda603b4e9a1", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "lean4_auditor", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 451, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1971, + "response_sha256": "85bf4906b2301a4acd9c0e762e019a91d999e2a95196bb05764e4990a748fa11", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 458, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1948, + "response_sha256": "bbe968e7accb65283cb6843a3aa597ff3831529e09212dec43a64421eb52279b", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 450, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1911, + "response_sha256": "483c0a0792a1d6637a5b41856997bd82de24ff2c8e3cee0f31b04322db5c337b", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec12_66", + "sec12_69" + ], + "retrieved_plane1_tokens": 1216, + "sealed_error_categories": [ + "lean4_auditor", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-22", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1664, + "total_elapsed_ms": 154513.4, + "total_prompt_tokens": 6543, + "total_retrieved_plane1_tokens": 4867, + "total_tokens": 8207, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 371, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1202, + "response_sha256": "b93aa701431fcf80d0dfa0104fe3358a1200f91e5661e909fad88fed1e8f2cc8", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 370, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1878, + "response_sha256": "d5949852aa1daa75b2a1a77ef3984f0c41abee055718cb548031392b8734566a", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 364, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1769, + "response_sha256": "4fa412ca69e34579bb7e57a0e993bd0f3e6c60a28ec84d8b225f211170b466d2", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec12_66", + "sec12_69" + ], + "retrieved_plane1_tokens": 1216, + "sealed_error_categories": [ + "lean_compile_failed", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 559, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1694, + "response_sha256": "c27a4976cd2db98a01203f72901c87d35ba3f04fff8943e6c4d161876970355b", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec12_66", + "sec12_69" + ], + "retrieved_plane1_tokens": 1216, + "sealed_error_categories": [ + "lean_compile_failed", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-23", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1579, + "total_elapsed_ms": 151923.68, + "total_prompt_tokens": 6552, + "total_retrieved_plane1_tokens": 4866, + "total_tokens": 8131, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 343, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1215, + "response_sha256": "ce7825e3df00ae4ac1d54271f42fd616409aa60fe0e3a357421af3213a7e1253", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean_compile_failed", + "missing_trust_words", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 398, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1788, + "response_sha256": "6b0f8bf3c0759798d284ac3419b644cace1d0a3e6bb7bddc7ca82b7cada7f408", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 419, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1786, + "response_sha256": "eecd51d964ddb77aa68ecd028fea9f2e3ba495c750221fb4494d3fedbedc49e3", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 419, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1763, + "response_sha256": "eecd51d964ddb77aa68ecd028fea9f2e3ba495c750221fb4494d3fedbedc49e3", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec12_66", + "sec12_69" + ], + "retrieved_plane1_tokens": 1216, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-24", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 1257, + "total_elapsed_ms": 133207.94, + "total_prompt_tokens": 6236, + "total_retrieved_plane1_tokens": 4862, + "total_tokens": 7493, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 270, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1211, + "response_sha256": "01cf04df9c61f8577c99fef6230fcea7957a690793e58246baa51d0025038288", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 272, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1537, + "response_sha256": "e39ffb2a53b314eaf080b0c394bffeca759e58f0f93b0a9bf822c319b0a6a17d", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 365, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1688, + "response_sha256": "5f2c447c7a724257745286b14f4e82b858c65913cdea3681a4bcc8f66336d4e4", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 350, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1800, + "response_sha256": "e01616b6e3a4ae5307e4de9f806ad8667aff3bfbb45d2d02f07bd73596956191", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + } + ], + "pass_curve": { + "k=1": { + "pass_rate_pct": 0.0, + "passed": 0, + "total": 96 + }, + "k=2": { + "pass_rate_pct": 1.04, + "passed": 1, + "total": 96 + }, + "k=3": { + "pass_rate_pct": 1.04, + "passed": 1, + "total": 96 + }, + "k=4": { + "pass_rate_pct": 1.04, + "passed": 1, + "total": 96 + } + }, + "total_items": 96, + "trajectory_any_pass_curve": { + "k=1": { + "pass_rate_pct": 0.0, + "passed": 0, + "total": 96 + }, + "k=2": { + "pass_rate_pct": 1.04, + "passed": 1, + "total": 96 + }, + "k=3": { + "pass_rate_pct": 1.04, + "passed": 1, + "total": 96 + }, + "k=4": { + "pass_rate_pct": 1.04, + "passed": 1, + "total": 96 + } + } + }, + "P2-B": { + "arm": "P2B_repair_loop", + "budget_by_k": { + "k=1": { + "completion_tokens": 65597, + "mean_retrieved_plane1_tokens_per_turn": 1200.0, + "oracle_calls": 96, + "prompt_tokens": 119806, + "replies_used": 96, + "total_tokens": 185403 + }, + "k=2": { + "completion_tokens": 126362, + "mean_retrieved_plane1_tokens_per_turn": 1202.9, + "oracle_calls": 179, + "prompt_tokens": 289269, + "replies_used": 179, + "total_tokens": 415631 + }, + "k=4": { + "completion_tokens": 235907, + "mean_retrieved_plane1_tokens_per_turn": 1199.0, + "oracle_calls": 336, + "prompt_tokens": 582253, + "replies_used": 336, + "total_tokens": 818160 + } + }, + "by_family": { + "A": { + "k=1": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=2": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=3": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=4": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + } + }, + "B": { + "k=1": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=2": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=3": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=4": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + } + }, + "C": { + "k=1": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=2": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=3": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=4": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + } + }, + "D": { + "k=1": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=2": { + "deployed_passed": 0, + "total": 24, + "trajectory_any_passed": 0 + }, + "k=3": { + "deployed_passed": 1, + "total": 24, + "trajectory_any_passed": 1 + }, + "k=4": { + "deployed_passed": 1, + "total": 24, + "trajectory_any_passed": 1 + } + } + }, + "deployed_pass_curve": { + "k=1": { + "pass_rate_pct": 0.0, + "passed": 0, + "total": 96 + }, + "k=2": { + "pass_rate_pct": 0.0, + "passed": 0, + "total": 96 + }, + "k=3": { + "pass_rate_pct": 1.04, + "passed": 1, + "total": 96 + }, + "k=4": { + "pass_rate_pct": 1.04, + "passed": 1, + "total": 96 + } + }, + "lora": [ + { + "id": 0, + "scale": 1.0 + } + ], + "outcomes": [ + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-01", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 301375.89, + "total_prompt_tokens": 6802, + "total_retrieved_plane1_tokens": 4782, + "total_tokens": 9874, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1195, + "response_sha256": "ca7efe4d4eb093ce3c8b7e957ed9e2d0297c2c4a763b9c448d42ce386b105ea9", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2086, + "response_sha256": "92e0790bc4bf5f409df1a03ee292625d2f105e5ce4f60e01a198e2c7ce5de2e1", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "UNSOUND_UNSAFE", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1418, + "response_sha256": "fe3c96195990a490e7c097ad5ce14adbc2ded2ceea769393f9c34018f4636b06", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_70", + "sec12_78" + ], + "retrieved_plane1_tokens": 1170, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2103, + "response_sha256": "7a35ffb67591b195cae9d39bddaf358fd1268dd3f17d24c03484dfe4910bb2f5", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-02", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 301617.14, + "total_prompt_tokens": 7363, + "total_retrieved_plane1_tokens": 4560, + "total_tokens": 10435, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1205, + "response_sha256": "e167c78ea8a6129ab0ae5a4af323389bb134da2b20d42717248e9a56264cb2e3", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2053, + "response_sha256": "ca5f50ecd265821d01c3402caa99e88577f2a29408693f97703b796d5d21fecf", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "NON_INIT_TACTIC", + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "NON_INIT_TACTIC", + "UNPROVED_SORRY", + "UNPROVED_ADMIT" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2096, + "response_sha256": "a3cb7750c29239bdc02891b3ba0c15d976a0cb6291b373693484193eb36cd43d", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2009, + "response_sha256": "e68b6f844b73f2aba0fc611cbe27637e153d9412cf09ee6a1a5d969904996aea", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-03", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 299871.43, + "total_prompt_tokens": 6912, + "total_retrieved_plane1_tokens": 4566, + "total_tokens": 9984, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1235, + "response_sha256": "33e30863c4997e20613371ba62dabc4680701cdc2106790e6261627ed9b26999", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "FORBIDDEN_META_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2108, + "response_sha256": "9a8340d9a561d874df53ed755c7438448f2fb76769c3100af81995413e7c5d5f", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "FORBIDDEN_META_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2125, + "response_sha256": "095c11869752ce9734b70f2d6d139be6ef00319a072528478210bd5377c860ff", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1444, + "response_sha256": "967bf06cfe500e865a8d84ec96c080a58120471293e7e41d62d772fe119f38fa", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_70", + "sec12_78" + ], + "retrieved_plane1_tokens": 1170, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-04", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2728, + "total_elapsed_ms": 218434.17, + "total_prompt_tokens": 7233, + "total_retrieved_plane1_tokens": 4629, + "total_tokens": 9961, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 424, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1149, + "response_sha256": "8469007ecfa633d722ba752acd89ca7dcc807e61677e93aa40b3ff250d219ab0", + "retrieved_chunk_ids": [ + "sec12_78", + "sec4_09", + "sec4_14", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1195, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "EXTRA_AXIOM_DECL", + "NON_INIT_IMPORT", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "TARGET_DEF_BODY_MUTATED", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1750, + "response_sha256": "4d6460a5fa5109c81f458d25b185f630e6c4d08adff993f1751ac683a368d3f0", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2205, + "response_sha256": "e74bf7675ee438948f0fe5992d089bbfab83a55dee7c1cae762c87396519f23e", + "retrieved_chunk_ids": [ + "sec12_66", + "sec9_43", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1154, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2129, + "response_sha256": "7ff66694d27275315a65749f4ded8298443a43485118b57a70e6e1be6238e247", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-05", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 308446.43, + "total_prompt_tokens": 7595, + "total_retrieved_plane1_tokens": 4528, + "total_tokens": 10667, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "UNPROVED_SORRY", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1228, + "response_sha256": "20e38c9d59fd3ea9309be2220dbd11771454b166d82575a0a815956633e0f155", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2182, + "response_sha256": "03da3927d916f18194ae960314ea4662a622bdac48b482c657ef5ba94afcbf1a", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "NO_KERNEL_THEOREM_FOUND", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2109, + "response_sha256": "38dde71899722bc54315e7e794e73f34344f3ba6c169db12d760d06bfd067921", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "FORBIDDEN_META_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2076, + "response_sha256": "c9c0f2f08a1d268461d47dc0995db96e7d617e3d713df69499b23449f60b551d", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-06", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 301767.67, + "total_prompt_tokens": 7780, + "total_retrieved_plane1_tokens": 4574, + "total_tokens": 10852, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1265, + "response_sha256": "1a012ff90853f6500bb47a80aec1ff658864341bce63a8124d9d99c24638b7ea", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2114, + "response_sha256": "031f4c4e1d7f160e9407d3d030cac9ca23552e6e81b145111e258a4546b5ef00", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "EXTRA_AXIOM_DECL", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2155, + "response_sha256": "7636f4b8fc157fada45fc304642c7d4dc311043cf2e659099f70d3ca0b844483", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2246, + "response_sha256": "a4b3dc5a7417085cff9e9d94d6bd8682610177f3a614f08721951e2c25df2bf5", + "retrieved_chunk_ids": [ + "sec12_66", + "sec9_43", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1154, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-07", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 302108.52, + "total_prompt_tokens": 7696, + "total_retrieved_plane1_tokens": 4624, + "total_tokens": 10768, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1251, + "response_sha256": "3a4290ddcb8cccab77e03c84ffc4e6771f2c2b3adf5838eb086396d38f212ffe", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2128, + "response_sha256": "7039023bc4ba401f4660916c4f0545be83c7e92799caee49376780485451bd7f", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2139, + "response_sha256": "d558da2d6a9e050b85aff8d689cacf9b86837c139bc42a29643598ade1d6d99e", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_trust_words", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2178, + "response_sha256": "23e5a2768363f836ddbd3fdd06dab8850be864b64fdedb38f5f9d25c4dbc25bd", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-08", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 301907.88, + "total_prompt_tokens": 7661, + "total_retrieved_plane1_tokens": 4560, + "total_tokens": 10733, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1281, + "response_sha256": "8f7e61c6ad1807c9b0779a21e6b4b1ceb6c4fb1ac1cb82fe1479d9736d058107", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2175, + "response_sha256": "02a22dc2875dcd7e932bd7e6c6d83f23344a6e88d3537a11a63029c662da6e37", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2056, + "response_sha256": "4f6a785226cc533b1a33b884b5b4c6e37389da778d24c01535fb048f13d88906", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2149, + "response_sha256": "bd3d9482662a36c92f8229f115f700dbd2dcf8544055c2851614d755572ff561", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-09", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2644, + "total_elapsed_ms": 233556.31, + "total_prompt_tokens": 7487, + "total_retrieved_plane1_tokens": 4624, + "total_tokens": 10131, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1298, + "response_sha256": "7c1ab9b0e350a7f732ac8b18423578def59ad3efe14fb21a72b5d1f84cd631e4", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2093, + "response_sha256": "48df9feea82868c2c595049fae74baf36ca9bf454432d3cf708529cecea72c36", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 554, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2095, + "response_sha256": "7c3bc9d9b78853b570a79e56afec24d809982d7aee89272dc361684fb0ea4b42", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 554, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2001, + "response_sha256": "7c3bc9d9b78853b570a79e56afec24d809982d7aee89272dc361684fb0ea4b42", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-10", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 2910, + "total_elapsed_ms": 244579.95, + "total_prompt_tokens": 7205, + "total_retrieved_plane1_tokens": 4770, + "total_tokens": 10115, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 606, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1291, + "response_sha256": "170f70fb66e0ca559dfaa24eeb9c82afa0f72a8bc0ef50aa6cb5b3ef01f517e5", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2183, + "response_sha256": "4dfabf8d0ccfc61d6abfb3461132af62fb3183896d07f1e3e2034578961f87ee", + "retrieved_chunk_ids": [ + "sec13_79", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1217, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2115, + "response_sha256": "a84d52fe4187155ef22f6872cf020b30ea1e1296bc28d0d4b0810ba896e6c73a", + "retrieved_chunk_ids": [ + "sec13_79", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1217, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1616, + "response_sha256": "22332e3d27310092a8c3e524c557d11fe7220e89e9ebdf889aba5baffe3be5e9", + "retrieved_chunk_ids": [ + "sec12_76", + "sec12_78" + ], + "retrieved_plane1_tokens": 1132, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-11", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 253905.27, + "total_prompt_tokens": 7872, + "total_retrieved_plane1_tokens": 4641, + "total_tokens": 10944, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1265, + "response_sha256": "dcaa7afce40e4779d64952b46e9a40e68a791c83da877f3b050e639b440a9f05", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec12_78", + "sec4_09", + "sec9_44", + "sec9_48", + "sec9_55" + ], + "retrieved_plane1_tokens": 1147, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "NON_INIT_TACTIC", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2187, + "response_sha256": "3e73410409eeb644e7cbd8c58c764847c9055064185a02c8bc9a9dfb4e351f3f", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_55" + ], + "retrieved_plane1_tokens": 1171, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2232, + "response_sha256": "c0f97fb679f9c5f2400fee78f9c39925229b5f638d31eba2fed9f565452cd3b7", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec12_74", + "sec12_78", + "sec9_44", + "sec9_48", + "sec9_55" + ], + "retrieved_plane1_tokens": 1152, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2188, + "response_sha256": "285e222f081538688cca5b7e88bbf940319685946b1214582b8693f859f70706", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_55" + ], + "retrieved_plane1_tokens": 1171, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-12", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 3072, + "total_elapsed_ms": 253738.97, + "total_prompt_tokens": 6266, + "total_retrieved_plane1_tokens": 4590, + "total_tokens": 9338, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1194, + "response_sha256": "ff6e3fd3d674a7e41dab81d2fe7dc275446ddfcdf0c84e5f3b6ef8d6c5f37e46", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1557, + "response_sha256": "5a45adfc9cb33ee5d7f263fe6c09cf0dffc66942edc47125f66e7858c9263de0", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1429, + "response_sha256": "76c463b894d65729d225785969dcfd373f142973f820828d2ccf25552048f0f3", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_70", + "sec12_78" + ], + "retrieved_plane1_tokens": 1170, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2086, + "response_sha256": "43e1d0d80a9dbcc059b102bc121d51d98a5682113570c6df5bd51bfcf23ceffd", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-13", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 253917.74, + "total_prompt_tokens": 7020, + "total_retrieved_plane1_tokens": 4542, + "total_tokens": 10092, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1234, + "response_sha256": "e46f4912ff2aa7008364dcffdc40995f07144995272762165e49dcc218fd93ac", + "retrieved_chunk_ids": [ + "sec12_78", + "sec4_12", + "sec9_42", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1162, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2090, + "response_sha256": "bddbd00b020357fd2153b0e933b67658e327336efbe2c4bf0ebe6fe8e634edda", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1487, + "response_sha256": "62fccc3dfe2934d976e4f136700d1993c584c58a99f0c34f32cfba9c5434c00a", + "retrieved_chunk_ids": [ + "sec12_76", + "sec12_78" + ], + "retrieved_plane1_tokens": 1132, + "sealed_error_categories": [ + "missing_trust_words", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2209, + "response_sha256": "9861778a85035733b239454856a0e7aab49434c744dfec558bd1169975babec8", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_trust_words", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-14", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 255350.58, + "total_prompt_tokens": 7026, + "total_retrieved_plane1_tokens": 4490, + "total_tokens": 10098, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1311, + "response_sha256": "e0aa2b2b8d4c8b1edd9d4ddabb766fb92ca82bdfbefabc6e5e200bb588cfe43b", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "FORBIDDEN_META_IMPORT", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2074, + "response_sha256": "27e4301fa886df2163aa5fd8def8f7bf34936e95d5e49bdb46a75eef8d2571e6", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "FORBIDDEN_META_IMPORT", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2073, + "response_sha256": "dd9631cd336aae46573b82f7d5dcacff9b6806d719dca9ce540604c686edf119", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1568, + "response_sha256": "962a5309fee5c1c9c0fc38fecc449cc9beed77814b3a7f961fc3956753e09c86", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_70", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1166, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-15", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3051, + "total_elapsed_ms": 250547.09, + "total_prompt_tokens": 7666, + "total_retrieved_plane1_tokens": 4773, + "total_tokens": 10717, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 753, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "UNPROVED_SORRY", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1226, + "response_sha256": "7e94bc8953ed222064a96e64bda0bb36268a4dc15a437f174140b0be136fa901", + "retrieved_chunk_ids": [ + "sec12_78", + "sec4_07", + "sec4_09", + "sec9_46" + ], + "retrieved_plane1_tokens": 1210, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 762, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2202, + "response_sha256": "e1da6144bb2e8812f31db1d2fe561ec763bae2a64af53e920379cdeb683d4f46", + "retrieved_chunk_ids": [ + "sec12_66", + "sec9_42", + "sec9_43", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1203, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2114, + "response_sha256": "38e7cf8bc21f2f194d903ebdada9fd618494e2e9f126f5423e1eaba9b92fd2ff", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2124, + "response_sha256": "9c297d82590c993a457408fbb2a8c9ef923ea1c219764c6a077344cfe582cc90", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-16", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 255559.75, + "total_prompt_tokens": 7350, + "total_retrieved_plane1_tokens": 4735, + "total_tokens": 10422, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1136, + "response_sha256": "d8536b0437a72c7fd70c3eb420724adec95da49eaa08a3cf170010a705fd535e", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_78", + "sec4_07", + "sec4_09", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1195, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_SORRY", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1995, + "response_sha256": "82774c70d40ea952c0649186960506c3f795bcf69bf08c66d16c3257de0a471d", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2154, + "response_sha256": "b24f8a730e9ae2d8c4d8062a805e25ba9cc27b8b11e231ef5dd15ec8fc9f9be3", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2065, + "response_sha256": "d7b7c00afc2a9c045308e70b3825e7f7448ddafc386f6e3acdc17d833a5cf892", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-17", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2993, + "total_elapsed_ms": 242592.68, + "total_prompt_tokens": 7481, + "total_retrieved_plane1_tokens": 4552, + "total_tokens": 10474, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1311, + "response_sha256": "3f882752222af894177962f391b1a55e0994e6d0826431b7d382c25cdaa1d38e", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2246, + "response_sha256": "aa9b613daaef1ad67d88605e6a664b34d351150dc9fb6de0aac731b6f9aa4b6f", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2272, + "response_sha256": "6e7ae16be4f33a35441989ea8294bd2b141ffeeca21264312c0005dbafb6a9c2", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_47" + ], + "retrieved_plane1_tokens": 1108, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 689, + "inference_diagnostic_rules": [ + "FORBIDDEN_META_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1652, + "response_sha256": "f4c14041755a4a48ec0f8fb6eb001b7206c1d57938cd181dd91575d76c553ede", + "retrieved_chunk_ids": [ + "sec12_76", + "sec12_78" + ], + "retrieved_plane1_tokens": 1132, + "sealed_error_categories": [ + "missing_trust_words", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-18", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246130.42, + "total_prompt_tokens": 6919, + "total_retrieved_plane1_tokens": 4694, + "total_tokens": 9991, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1261, + "response_sha256": "f2717ff693ef6a3aa3a69244af429c2ff44a165de7e4dc9a498713a8b782176c", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_74", + "sec4_09", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1174, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2091, + "response_sha256": "b8e2141bd2ca008e78f3c474f54e2f0ec78c98b05e69fe0ae027a0a634d66d04", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_74", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1175, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1497, + "response_sha256": "af619761c84d21cbc9bfb4456dc3803ad6fa91fd21240f979a38ccf0dc5f36b6", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_70", + "sec12_78" + ], + "retrieved_plane1_tokens": 1170, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "SYNTAX_OVERRIDE_DECL", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2070, + "response_sha256": "8913cd3359f01aec844ccb48b9747428c2c89a72a07cb28a7e34f2fd4f1417d5", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_74", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1175, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-19", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246781.14, + "total_prompt_tokens": 7681, + "total_retrieved_plane1_tokens": 4686, + "total_tokens": 10753, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1308, + "response_sha256": "cf7ee03d45911ad24d7399bc851bc0427e3a25a782e0dd610fd85994e76ef31e", + "retrieved_chunk_ids": [ + "sec12_78", + "sec4_12", + "sec9_42", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1162, + "sealed_error_categories": [ + "missing_trust_words", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2003, + "response_sha256": "76eb30575217bc8ff17368aaa8891e51de961e7483993969a4353569dc9b2390", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_ADMIT", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2191, + "response_sha256": "ed1cd1cd43656027f9adf9dade2449d96ab3041a3775f8637416d111323f2b25", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2179, + "response_sha256": "e9ead6e7553463cce0bb7fa2a401cf45a5dac06509aa94ad5505c556896eae6a", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-20", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246717.9, + "total_prompt_tokens": 7333, + "total_retrieved_plane1_tokens": 4663, + "total_tokens": 10405, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1340, + "response_sha256": "1ee9f8d6599f65d347df4884c8343daaed01b7430743eab3770e58c0cbdcc7ba", + "retrieved_chunk_ids": [ + "sec12_78", + "sec4_12", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1144, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2195, + "response_sha256": "8b591d8fbcee28e0d4d6ae31a3f2ae7e126346f273451b7756c12a201c5dcb21", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1546, + "response_sha256": "c597f9b708a5182299ff6cd362582b0540dfe11a342d5a27313a91614526eccd", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_70", + "sec12_72", + "sec12_74", + "sec12_78", + "sec4_09" + ], + "retrieved_plane1_tokens": 1159, + "sealed_error_categories": [ + "missing_trust_words", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2252, + "response_sha256": "63a59aff99820b6a8db8764960fa25fd0422d39be0a4c17cee5307f87ce888ba", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-21", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 247574.24, + "total_prompt_tokens": 7906, + "total_retrieved_plane1_tokens": 4801, + "total_tokens": 10978, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1333, + "response_sha256": "58f87dd67de67ee524986017983a59f4223b7e6750b8ed21c4bf868c69b6ced4", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_78", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1213, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_IMPORT", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2223, + "response_sha256": "0faf624f3fd1d1df54eaf1461381e79e9d82718f5d8465e6fd431b9fe460a17e", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_trust_words", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "FORBIDDEN_META_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2115, + "response_sha256": "4e438fab4bbc4927d01d158e36b90e0031df6d0d3146cde81d636beeab12583a", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "FORBIDDEN_META_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2235, + "response_sha256": "9ef9d35d2f5914f471c8dd3157ad4f909e554dec8884ec16a7f6ed8b73843ffc", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-22", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 2883, + "total_elapsed_ms": 231896.51, + "total_prompt_tokens": 7560, + "total_retrieved_plane1_tokens": 4777, + "total_tokens": 10443, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 579, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1317, + "response_sha256": "4be30fdf1ff49aeb0be1c6dfcd7a0ae846acf06ac3c8dd74a6c671205d75840c", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_78", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1213, + "sealed_error_categories": [ + "missing_trust_words", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1987, + "response_sha256": "ffda576e97eb20e707d404aef0f9a66d58eded220afb09fe4d32c8420a34f576", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2131, + "response_sha256": "7253e163dc4f51a7d604e0275b250c5a77ba2519d32149cdd2f8ec34e43215a5", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2125, + "response_sha256": "cf06cc342907a70cded6bbd2056f656688d620f1dae6375b780f5cdefcda4324", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-23", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 247691.72, + "total_prompt_tokens": 7268, + "total_retrieved_plane1_tokens": 4624, + "total_tokens": 10340, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1278, + "response_sha256": "fae8135b9935ebfb6c20799590809a961b56f15d43dcd12b4287df37184c06c1", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "SYNTAX_OVERRIDE_DECL", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1741, + "response_sha256": "22edceb30c5993889aa19fb7271d16bef847cc50643cff44db7cc9487343f2c8", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_45", + "sec9_48" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "UNPROVED_SORRY", + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2101, + "response_sha256": "b29cbf78980d7c6ef2c5a032210a8b3fe3fdf55d63233926f41295a354942b3b", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2148, + "response_sha256": "3bb036c7790d65e88d067c4d99c896342942f52ff1f832f0d4ef0e94fc3377a9", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1140, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "A", + "id": "HOLD-A-24", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246868.2, + "total_prompt_tokens": 7788, + "total_retrieved_plane1_tokens": 4715, + "total_tokens": 10860, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NON_INIT_IMPORT", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1275, + "response_sha256": "895a8f3997c9833cb78d44c586ce8961617fbaa339b3b03c12752261b323a0f3", + "retrieved_chunk_ids": [ + "sec12_78", + "sec4_09", + "sec4_12", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1152, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2209, + "response_sha256": "ca37bd1fd1e4d04da722a17f778423c8cdf82de99d2c34fec19fc39e3b7b29d8", + "retrieved_chunk_ids": [ + "sec12_66", + "sec9_42", + "sec9_43", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1203, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2180, + "response_sha256": "164da56ea1286967e3ce6a3469a560f55c2ed8b1d7de7b5627f2011dc2cf5232", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2124, + "response_sha256": "58a4ca99b896e962d61195099654e80524672eb3f6392368be17cf5fa364edd6", + "retrieved_chunk_ids": [ + "sec12_78", + "sec9_42", + "sec9_44", + "sec9_45", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1180, + "sealed_error_categories": [ + "missing_or_empty_section", + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-01", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 3072, + "total_elapsed_ms": 236843.97, + "total_prompt_tokens": 6469, + "total_retrieved_plane1_tokens": 4833, + "total_tokens": 9541, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1299, + "response_sha256": "c68384ecca2001822f6ca72c06a1e98bd59b066edcae0e333c96e3980c444710", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2072, + "response_sha256": "d3a09b076ff55d3abeb4e2a5c780516e3049320c1b0e63180fd4506789f71743", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1569, + "response_sha256": "988bbd35161086b3e26f1f5d87602cd15b5f26095aa061e59c8afd39d9e7fea5", + "retrieved_chunk_ids": [ + "sec12_69", + "sec12_72", + "sec5_20", + "sec6_30" + ], + "retrieved_plane1_tokens": 1203, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1529, + "response_sha256": "a1d24fb293036451c51d513aad9d0da97d48eb429fff4756fbc31b9a72ac7e86", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_68", + "sec12_74", + "sec6_27" + ], + "retrieved_plane1_tokens": 1205, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-02", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 236911.01, + "total_prompt_tokens": 7195, + "total_retrieved_plane1_tokens": 4863, + "total_tokens": 10267, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1239, + "response_sha256": "b832e6309ad6080f6465b6accadb7e2e4f6bca379a77fa4283478b9aa018b217", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1989, + "response_sha256": "1806e849b33355896301305aa2fdd30f0e4332c88d911aa485d8006a890b88f8", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1976, + "response_sha256": "9a8941e2f0325a5f665dd9bd5c009193807606152e9ef1af43128fa038bd3d8c", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1991, + "response_sha256": "3bde67fb75e4e7c4ddc101b5bcaa046cda51703fb4e40c7ea3e7a39e6ad9e234", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-03", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 236980.19, + "total_prompt_tokens": 7547, + "total_retrieved_plane1_tokens": 4863, + "total_tokens": 10619, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1274, + "response_sha256": "c52178c74c2c2aff256921ea5721158b33a714a98227f08ba49304505c270431", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2182, + "response_sha256": "dc4e593be7292e7a31cacc4b7ef3c58fd97414b7a0109afb0d0fd342435684ee", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2034, + "response_sha256": "4d1ba12f9612ad133a1bd284ea6d4f6e78b6e05c4b7b8e117f0a5f92ed561863", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2057, + "response_sha256": "9f76e8f664e19e660fe94b7f3d5d9371dd2ddd66de96a448a838259d86a2e4b6", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-04", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 237020.31, + "total_prompt_tokens": 7615, + "total_retrieved_plane1_tokens": 4846, + "total_tokens": 10687, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1347, + "response_sha256": "fb5029aefd48fcb09c4ea5ab784af3e398c357f8e77caa9954238d4aa1889b80", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2149, + "response_sha256": "8a7ceb49f3154cbb8ac57841d71566bd58b2af7f3a1ba4260daaeba337fbb878", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2117, + "response_sha256": "f8b3b76f24bf4cd47f09cb51ec2983a9eaaf1bd851710a0d5f605f1b2aa5f3ba", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2002, + "response_sha256": "937cddc15e3c9089d49f5f2d035d7de3c14e80c41b2371a5f2da0ee05fd4b425", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-05", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 237059.21, + "total_prompt_tokens": 7407, + "total_retrieved_plane1_tokens": 4859, + "total_tokens": 10479, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1272, + "response_sha256": "cc8c1d7543386bd9497d7c89cc7be0f5c5b3d65f36b8959a96c64080d0b3a551", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec6_25", + "sec6_26", + "sec6_30" + ], + "retrieved_plane1_tokens": 1202, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2153, + "response_sha256": "a1b0a0be23e3a5437cef952e56a9f2a0f7eef4223ca393377e40cb82fd54203a", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2031, + "response_sha256": "d2e1af0a2963752fd37b36752b4ac64aadff99a6cad0c96e680c42277ac21b45", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1951, + "response_sha256": "d7b2118f199d183779ffa5a30fe530a03807f6e5a486ef728f5c4e2661bffcb7", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-06", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 237120.99, + "total_prompt_tokens": 7598, + "total_retrieved_plane1_tokens": 4839, + "total_tokens": 10670, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1456, + "response_sha256": "11159947c92038d3bf240d3bd55cc8e8deda56bb3915ee5ddf03b431e31de392", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2113, + "response_sha256": "656fd498eec1ca1c0edb0ed7c692ef703b425b74bd31b4b13b0d7d9b2e9539b3", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1711, + "response_sha256": "b458ac10296f0567cdc5e8d5f43c3eaa0851d55a565316a9917cd580cf80c6c9", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2318, + "response_sha256": "5781cf0f733d849f67250d2d6849e90850d69d2cb0cf4dac3474147d600102dc", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-07", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 237108.26, + "total_prompt_tokens": 6392, + "total_retrieved_plane1_tokens": 4840, + "total_tokens": 9464, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1290, + "response_sha256": "33e2e0467e441e863e8d3290a9bd6c006ce82df13590c22b9c4552c26d355a5d", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1989, + "response_sha256": "91fcbe0f469078939845920b0d9e7590a2fcdc23a5a5b4ebac0713dbbd4ca1bc", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1519, + "response_sha256": "e0db0d5e7693aa0c3aa3b3b22121e2322c576cb1d9d84694f5cc2e27355baf06", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1594, + "response_sha256": "40e1408033fc11273b77d96aab0b701ce8627e95a636e96741f49ac3c60412b3", + "retrieved_chunk_ids": [ + "sec12_72", + "sec12_74", + "sec5_20", + "sec6_30" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-08", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 237140.23, + "total_prompt_tokens": 7816, + "total_retrieved_plane1_tokens": 4814, + "total_tokens": 10888, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1298, + "response_sha256": "4d5a9448002706616ae223f742c4c713f26cdf4e4d7ae6ee1bdac04c866dcf9c", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec6_25", + "sec6_26", + "sec6_30" + ], + "retrieved_plane1_tokens": 1202, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2146, + "response_sha256": "afd25c63e11ca05544905e6e48d26caeb6739f627d91803b00458f5ee90398d0", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2187, + "response_sha256": "cfc90aa4671f3f8ae0d186147fa7c610c8c62cc1deb57830dbc3a7937b5b2885", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2185, + "response_sha256": "c3e22b347d2720b94fc4a102c0037f51eec91407ae5794ef65db1377d8089c4f", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-09", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 234893.32, + "total_prompt_tokens": 6403, + "total_retrieved_plane1_tokens": 4846, + "total_tokens": 9475, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1278, + "response_sha256": "85e342afa8d1022a641e8dc46547b17ee8cdece78879ed52ff8b657a6a5549bb", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2162, + "response_sha256": "85e342afa8d1022a641e8dc46547b17ee8cdece78879ed52ff8b657a6a5549bb", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1543, + "response_sha256": "7ee2d5ba6755b902fbb3ead841c407daecac36f736d787a2978dc0e66f9bcf18", + "retrieved_chunk_ids": [ + "sec12_69", + "sec12_72", + "sec5_20", + "sec6_30" + ], + "retrieved_plane1_tokens": 1203, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1420, + "response_sha256": "d017d226bf651c63e5781543d3c93cf99ff71031ca2c522c411dd2ef3b70db2d", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1218, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-10", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 234958.52, + "total_prompt_tokens": 7604, + "total_retrieved_plane1_tokens": 4863, + "total_tokens": 10676, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1306, + "response_sha256": "7aae6384fa35cf1f9b455803011a747159d1fe8c223e1274e10f13d453784968", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2158, + "response_sha256": "de7b13c519e302145317336d34c509d9469cc2269ea60f0664c81567bedab1dd", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2120, + "response_sha256": "b1bed5b06b92c04b3826f49ac956be2afc3fbbd739fe598cfef9da79e0861142", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2020, + "response_sha256": "c10bd44713f0e0785b191112810d613ba9d685b31d8ab733e822781540b0b1b7", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-11", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 3072, + "total_elapsed_ms": 234993.13, + "total_prompt_tokens": 6341, + "total_retrieved_plane1_tokens": 4841, + "total_tokens": 9413, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1346, + "response_sha256": "1a9b57dbf7924f8312bf79f08989d0fb31cc5c5cd10012bcd476a6f4abcb15d7", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2099, + "response_sha256": "8a48bd8021658ade95e102a019744586d44b7444f56506f00e88ca3c1f9fe580", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1463, + "response_sha256": "b039fa545a8033acde0ef329c4747489d8f976986030e09f7b172ae2164e6386", + "retrieved_chunk_ids": [ + "sec12_69", + "sec5_15", + "sec6_26", + "sec6_31" + ], + "retrieved_plane1_tokens": 1185, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1433, + "response_sha256": "2c299e20781269c834726a0ddef8118feeb714e8a731c4e2fc81b8e61014dc3c", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1218, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-12", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 235032.31, + "total_prompt_tokens": 7274, + "total_retrieved_plane1_tokens": 4876, + "total_tokens": 10346, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1292, + "response_sha256": "d29b7c09f12531a9ae0b8355f9bfd3a64efe4a9a4d98eee8eb7858733fef69a2", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2002, + "response_sha256": "1ffd38e99adee447bf76e1f89ca8000b4506f4470eda8effe21671cecbb271e9", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1966, + "response_sha256": "5102c6c664279129f7b3339f5b8067afc1246d11314fcb14b608ed4314f36ad8", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2014, + "response_sha256": "ea8149b1af3ae54ed92be769466c64293e0516df861b42c8fb9ce5ca1f1dd77b", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-13", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 235078.75, + "total_prompt_tokens": 6918, + "total_retrieved_plane1_tokens": 4840, + "total_tokens": 9990, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1439, + "response_sha256": "00f80982c27bbb85f760c658921f09f4594215d6615e43db8bf1e62a7b04f2e2", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2260, + "response_sha256": "00f80982c27bbb85f760c658921f09f4594215d6615e43db8bf1e62a7b04f2e2", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1624, + "response_sha256": "d520edbf828280f2e2ffa7255906d397b39550d87194acab23c7fe3ef20b7362", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1595, + "response_sha256": "e5b311800297162083ef02ea7bca1e5041c0a9e32e868ff0c98602ffa47d6d9e", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_72", + "sec12_74", + "sec6_26", + "sec6_30" + ], + "retrieved_plane1_tokens": 1211, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-14", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 3072, + "total_elapsed_ms": 235098.79, + "total_prompt_tokens": 6914, + "total_retrieved_plane1_tokens": 4846, + "total_tokens": 9986, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1410, + "response_sha256": "9cc723ca34ce89e66d04e21c4053a2915df761dd38bf83647a9a001a063206a2", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2267, + "response_sha256": "30a1b76aaf279b20b754b267693efb29717d1d149d395af1ef21c3e8653ea232", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1679, + "response_sha256": "515a8f9e7835f2b0e9d22456fadcb8dbc3b91fb36901b791881280dd41f8c6e2", + "retrieved_chunk_ids": [ + "sec12_69", + "sec12_72", + "sec5_20", + "sec6_30" + ], + "retrieved_plane1_tokens": 1203, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1558, + "response_sha256": "33d74a2ab11d5b1dd3050bd0ae701eadd323cd4445caecc1f3cb9d83dc3aceca", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1218, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-15", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 235138.86, + "total_prompt_tokens": 7535, + "total_retrieved_plane1_tokens": 4849, + "total_tokens": 10607, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1487, + "response_sha256": "06363b6d6dd6785aa1df107142a007b90f23b20c3b09f03552cfb905955c4429", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2260, + "response_sha256": "ef9a4dcbce90b39ad67dad21669860381f5bb6f60f65dcaf35f6a3de30854bee", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2152, + "response_sha256": "3f68774915a5ad3414fd1f8ac2fe4131e07a6b7a2ebaf23eb8057f745e437325", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1636, + "response_sha256": "9969ce96dbd92eefd01ba2de0284ac26939a54a3481f89082feb9506d134c66f", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_69", + "sec12_72", + "sec6_26", + "sec6_30" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-16", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 3072, + "total_elapsed_ms": 235151.82, + "total_prompt_tokens": 7558, + "total_retrieved_plane1_tokens": 4859, + "total_tokens": 10630, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1323, + "response_sha256": "883c21f0846157cb71d87f47efe825271e437c258cc4e731a62fb0aebb5362d6", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec6_25", + "sec6_26", + "sec6_30" + ], + "retrieved_plane1_tokens": 1202, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2075, + "response_sha256": "5a2c8afcc41f975f9acb15c7558b274ae25c6108d0323898fa12410b49e472ba", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2035, + "response_sha256": "4ea6b6302208bcaaff6c5814855b4327b909c9676d3501a3552b66697dca256e", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2125, + "response_sha256": "b644444cd3af14a41027d3a8d0756264b54139cb8df87f18a76ca5d442fa6eb4", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-17", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246415.58, + "total_prompt_tokens": 6925, + "total_retrieved_plane1_tokens": 4834, + "total_tokens": 9997, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1393, + "response_sha256": "d6e0718bf733d69f4db519e24b6c8415af8fc8ec563b9c507cef43f1da73ecab", + "retrieved_chunk_ids": [ + "sec5_16", + "sec5_20" + ], + "retrieved_plane1_tokens": 1208, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2088, + "response_sha256": "572f7a0a17810027ffe5ddff10707b5a3f8a92b678901f58127576569eb48e9f", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1435, + "response_sha256": "419d3ac2f0b7a66371585b44f50838febb27ceab6bbba6028782f88aeb51868a", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1218, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2009, + "response_sha256": "03d7d7f9fd76773f0b2267b91552b3eee181c7c4af45b70e24e714ac64826115", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-18", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246467.54, + "total_prompt_tokens": 6742, + "total_retrieved_plane1_tokens": 4847, + "total_tokens": 9814, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1253, + "response_sha256": "19ae419c6c27fc1ecc22a8fba70adb0e58c8c65e7adbadede45f142aa61a1fd1", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1968, + "response_sha256": "3172f4bd6432b767e83c68cff7e3a2bcbc1032e3f56e139b25e105768fe4a4e5", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1970, + "response_sha256": "89ffa314a8d597debb31e96dd1435e3e1eff6d88384ad04d841624b26fe1fa08", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1551, + "response_sha256": "57bb3e06454ef545424982f40ffe59add550b87417ca018cddd1a7d5f1c131f4", + "retrieved_chunk_ids": [ + "sec12_69", + "sec12_72", + "sec5_20", + "sec6_30" + ], + "retrieved_plane1_tokens": 1203, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-19", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246524.65, + "total_prompt_tokens": 6462, + "total_retrieved_plane1_tokens": 4848, + "total_tokens": 9534, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1382, + "response_sha256": "8e505b2d920b965a6f7108488b00f9c4ca635ba7c8d454937f25032a2e22442c", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2092, + "response_sha256": "b19890ecba4824d29b2f6f67b627bbded449f06fa14068e53e29443621a688d5", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1503, + "response_sha256": "c4eddbfe336bca8c77516053154068af7abdcc08b78a9e9c2c28567f85ba7345", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_69", + "sec12_72", + "sec6_26", + "sec6_30" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1485, + "response_sha256": "b88347422dc96c9f759af6b637f7521f1b3e9e527e96ae7533ce20322a5e98e4", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1218, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-20", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246588.44, + "total_prompt_tokens": 6429, + "total_retrieved_plane1_tokens": 4863, + "total_tokens": 9501, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1226, + "response_sha256": "cd9f18839077fc8efdd78316e1c888020a9e0c80f39b776db85d98d6c7059429", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1218, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1874, + "response_sha256": "2bbeef230c0881c9b7ec5641019c3c35bef56842e016e22601071f6ccdf64b04", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1218, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1463, + "response_sha256": "f91dec183be944c0e443e2723ce259f9d8f35d9e585ff8e1cd0dc873945f1526", + "retrieved_chunk_ids": [ + "sec12_65", + "sec5_15", + "sec5_20" + ], + "retrieved_plane1_tokens": 1209, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1866, + "response_sha256": "c08e67a635959023d78da3430eb3443c6b0a94ee9197c0d37037e7aacfc99cae", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1218, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-21", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246683.77, + "total_prompt_tokens": 7277, + "total_retrieved_plane1_tokens": 4795, + "total_tokens": 10349, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1398, + "response_sha256": "7d9e2d18a204be07abb5cfdb8565b70234fc79192a354f9e17f2f453dd2155e8", + "retrieved_chunk_ids": [ + "sec11_61", + "sec12_77", + "sec5_15", + "sec5_16", + "sec6_25" + ], + "retrieved_plane1_tokens": 1196, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2240, + "response_sha256": "853d1dc9df5dc77d6432c506ef308df722024441270c0aee3b10cfc9c9bf42dc", + "retrieved_chunk_ids": [ + "sec11_61", + "sec12_77", + "sec5_15", + "sec5_16", + "sec6_25" + ], + "retrieved_plane1_tokens": 1196, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1590, + "response_sha256": "4c455fccacdb378a8f7a7f031bd283c18c81966bc7ed230e03dd6438c58218cb", + "retrieved_chunk_ids": [ + "sec12_77", + "sec6_27" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2049, + "response_sha256": "70d2daab46ac62d13fbc6e97ef731ee7e16f4d13ace29a620300b4bc66b36885", + "retrieved_chunk_ids": [ + "sec11_61", + "sec12_77", + "sec5_15", + "sec5_16", + "sec6_25" + ], + "retrieved_plane1_tokens": 1196, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-22", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246682.95, + "total_prompt_tokens": 7027, + "total_retrieved_plane1_tokens": 4848, + "total_tokens": 10099, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1364, + "response_sha256": "6d9e6496813e807abd3b4c89aca022c1aa3e061bf7fccdfb107345922c5f9185", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2052, + "response_sha256": "c7d4646ec89af2c2822c0a06e68a3056b15686863c7a4c8deeae4e0500338ba9", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1570, + "response_sha256": "58833b3e026748f093a7e5fd0fe1288040f0ef31b22164a1fe82d5845895f61a", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2041, + "response_sha256": "54178c6585cf0a1e8986212712cf29a33185a9e8525154b77ceab12ffb64334d", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1219, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-23", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 768, + "total_elapsed_ms": 61296.53, + "total_prompt_tokens": 1291, + "total_retrieved_plane1_tokens": 1206, + "total_tokens": 2059, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1291, + "response_sha256": "c5a2851b4fbf03b04f9705d176452e6885b107e497b70f9ce2e828be2b3e88dc", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec6_26" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "B", + "id": "HOLD-B-24", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 246708.43, + "total_prompt_tokens": 6258, + "total_retrieved_plane1_tokens": 4840, + "total_tokens": 9330, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1307, + "response_sha256": "a62bf5ae3b77c2099fb089b60230a811b317ed62f489f41459dc039ead505352", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1987, + "response_sha256": "a386d90e22d3daa10300af5cac6ae9da6e7e410aedbcc1dde99f384238346b20", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1496, + "response_sha256": "1872d0218657f46ee6fa805d8c3357e8a2c2951b92930f59aadb8cf6b1f80bc0", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1468, + "response_sha256": "03e089827ec89fabf1b7f558cbdba7bf97ce4487e44444ec75b0c72ab8b947fc", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_72", + "sec12_74", + "sec6_26", + "sec6_30" + ], + "retrieved_plane1_tokens": 1211, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "syntax_error" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-01", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 354, + "total_elapsed_ms": 29289.99, + "total_prompt_tokens": 1103, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1457, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 354, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1103, + "response_sha256": "994f6eff40dea26542c3c33292f34b65e25ac72f67184f5639ace15790678f2c", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words", + "missing_expected_tier_label", + "missing_required_boundary_fact" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-02", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 341, + "total_elapsed_ms": 28437.17, + "total_prompt_tokens": 1046, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1387, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 341, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1046, + "response_sha256": "aaa65cc662eb8491b66a7cec4d94b8c8c0ea1d5ad23eeb122e787bbf723732e6", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words", + "missing_expected_tier_label", + "missing_required_boundary_fact" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-03", + "inference_certified_at_turn": 2, + "oracle_calls": 2, + "stagnation_events": 0, + "total_completion_tokens": 1258, + "total_elapsed_ms": 77766.98, + "total_prompt_tokens": 2703, + "total_retrieved_plane1_tokens": 2408, + "total_tokens": 3961, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 490, + "inference_diagnostic_rules": [ + "INCOMPLETE_TRUST_BOUNDARY" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1073, + "response_sha256": "de0f1d5ef5871ae4d5063ff93f57c4e37686da7bf647293c9f8cc1ce82535d64", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "missing_required_boundary_fact" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1630, + "response_sha256": "d774844185e4a0a8d7e6cce5b7ea23455791ad40b7bf0c7d3434af6cf564312b", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "missing_required_boundary_fact" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-04", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 277, + "total_elapsed_ms": 24873.22, + "total_prompt_tokens": 1059, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1336, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 277, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1059, + "response_sha256": "a5ac7d38976f5d39214192b271540e9192d1f024faa46ae080e34496ceef676e", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-05", + "inference_certified_at_turn": 2, + "oracle_calls": 2, + "stagnation_events": 0, + "total_completion_tokens": 926, + "total_elapsed_ms": 65571.51, + "total_prompt_tokens": 2536, + "total_retrieved_plane1_tokens": 2408, + "total_tokens": 3462, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 365, + "inference_diagnostic_rules": [ + "INCOMPLETE_TRUST_BOUNDARY" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1052, + "response_sha256": "9b3880f211b4bb3c41e0a31a05e0e9b42c574c2c82924e7e211a9820a91aa482", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "missing_expected_tier_label" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 561, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1484, + "response_sha256": "ef70552cbf649993542a6cd8353e07bcd4a87ad0759039b409abc33e47856ed4", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "missing_expected_tier_label" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-06", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 514, + "total_elapsed_ms": 41438.86, + "total_prompt_tokens": 1053, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1567, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 514, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1053, + "response_sha256": "e7c6b804bbfcb4c0a05ebcdcdd36949682db316a4e9a5b47ff21ed483f19633b", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-07", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 414, + "total_elapsed_ms": 33946.67, + "total_prompt_tokens": 1061, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1475, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 414, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1061, + "response_sha256": "074f4c74e65c0f64b1414280fe2813f7bbe85c33c96154c52a78bb31164f49c7", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-08", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 112254.71, + "total_prompt_tokens": 5942, + "total_retrieved_plane1_tokens": 4812, + "total_tokens": 9014, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1036, + "response_sha256": "2d9f9063d36e37d513a832890b67d7063705a6755387a7626c01cb30cd42acf2", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "INCOMPLETE_TRUST_BOUNDARY" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1819, + "response_sha256": "fb069a847790709292d6d1abad8e02e4c647e60f648bba607644f9321c6b9a6f", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "INCOMPLETE_TRUST_BOUNDARY" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1759, + "response_sha256": "6ba93f087e17abebe6291c3246eb31ee0261c8709d49e46d3407da17161d4896", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1328, + "response_sha256": "9da15c3f4551699ad5143c51f49c177ead56f4fe97cef71a063e82158cfdae41", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_74", + "sec12_78", + "sec8_39" + ], + "retrieved_plane1_tokens": 1200, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-09", + "inference_certified_at_turn": 2, + "oracle_calls": 2, + "stagnation_events": 0, + "total_completion_tokens": 1255, + "total_elapsed_ms": 102505.75, + "total_prompt_tokens": 3321, + "total_retrieved_plane1_tokens": 2446, + "total_tokens": 4576, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 487, + "inference_diagnostic_rules": [ + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1261, + "response_sha256": "b038cbc11f8989f365234365eed4c12d51d3a0a2dee7bd7ea38d48545853aca4", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 2060, + "response_sha256": "ac5a8cc9f80e172adf1098a47e06f683c6cd63911816c5c88f3c453795f493d3", + "retrieved_chunk_ids": [ + "sec4_09", + "sec5_15", + "sec8_39", + "sec8_41" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_trust_words", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-10", + "inference_certified_at_turn": 2, + "oracle_calls": 2, + "stagnation_events": 0, + "total_completion_tokens": 1282, + "total_elapsed_ms": 104004.58, + "total_prompt_tokens": 3375, + "total_retrieved_plane1_tokens": 2450, + "total_tokens": 4657, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 514, + "inference_diagnostic_rules": [ + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1266, + "response_sha256": "ae28c628e716d7491ecd60ec9eac5e72ddc73677f1719906aac78a0c8ef4eb64", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_expected_tier_label", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 2109, + "response_sha256": "0110534de5aa24aca535b3f9e69a403f6f5250cb9a8b53f9824b1a965b59957e", + "retrieved_chunk_ids": [ + "sec8_39", + "sec8_41", + "sec9_42" + ], + "retrieved_plane1_tokens": 1224, + "sealed_error_categories": [ + "missing_trust_words", + "missing_expected_tier_label" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-11", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 2819, + "total_elapsed_ms": 196784.55, + "total_prompt_tokens": 7762, + "total_retrieved_plane1_tokens": 4892, + "total_tokens": 10581, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 565, + "inference_diagnostic_rules": [ + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1276, + "response_sha256": "0d9cdaf60adf6ecca33838b6a8f9c0f63da3da6a69e2518ff41c4366dced4bba", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2144, + "response_sha256": "a5fa5065433e23774ed68a74da199b47057050b60b3796f8421522213dee2356", + "retrieved_chunk_ids": [ + "sec4_09", + "sec5_15", + "sec8_39", + "sec8_41" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 718, + "inference_diagnostic_rules": [ + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2150, + "response_sha256": "211c5a547cd39053e4315c6d1b17941ee27381f13150ab655976033f85df5bea", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2192, + "response_sha256": "4296a6b221d53c9157b308b51d121b0bece793335592adb922b84964056e83e9", + "retrieved_chunk_ids": [ + "sec4_09", + "sec5_15", + "sec8_39", + "sec8_41" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-12", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2874, + "total_elapsed_ms": 198876.25, + "total_prompt_tokens": 6836, + "total_retrieved_plane1_tokens": 4877, + "total_tokens": 9710, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1263, + "response_sha256": "d7105f3628e2103efaf9144c72920689648737f82092959a817ec875f4611bac", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words", + "wrong_declared_tier", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2110, + "response_sha256": "3b5b155132cc61fa9ff1be0615b9a9afbed23f130a4aae851b7c0ee5ff7ccf4c", + "retrieved_chunk_ids": [ + "sec4_09", + "sec5_15", + "sec8_39", + "sec8_41" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_or_empty_section", + "wrong_declared_tier", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1945, + "response_sha256": "b75fe9fcd3e8d787739f8b6ba2fb09fb924fd78aaf7cea19695d01402662f855", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_or_empty_section", + "wrong_declared_tier", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 570, + "inference_diagnostic_rules": [ + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1518, + "response_sha256": "f012b823407fc8507620c3472bc4fb887fda20196c3e7ac77b6eba030b93a9b9", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_74", + "sec12_78", + "sec7_32", + "sec7_34" + ], + "retrieved_plane1_tokens": 1205, + "sealed_error_categories": [ + "missing_expected_tier_label", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-13", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2842, + "total_elapsed_ms": 197749.75, + "total_prompt_tokens": 6955, + "total_retrieved_plane1_tokens": 4837, + "total_tokens": 9797, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1356, + "response_sha256": "cd4100a68a40fbf19cfc2f9fbd362b6f48d29322c09c4e68665c093a403bc932", + "retrieved_chunk_ids": [ + "sec7_34", + "sec8_36", + "sec8_39", + "sec9_48" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "missing_required_boundary_fact", + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1969, + "response_sha256": "5e64e674d082141fa2d8ccd9db6e49b7dda2a8e615eff140b59770ce7cbe67d4", + "retrieved_chunk_ids": [ + "sec4_09", + "sec6_25", + "sec7_32", + "sec7_34", + "sec8_39" + ], + "retrieved_plane1_tokens": 1211, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 538, + "inference_diagnostic_rules": [ + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1525, + "response_sha256": "a71ca82fa9286fa7e5bb1e5d33ab09dba97e24bcc0ca67c5e5df729ddf47f449", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec12_78", + "sec7_32", + "sec7_35" + ], + "retrieved_plane1_tokens": 1185, + "sealed_error_categories": [ + "missing_trust_words", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2105, + "response_sha256": "81628662020007071a9c711d6aa4b830f88ac9bb35e5ebfa97575e842f1dc1ab", + "retrieved_chunk_ids": [ + "sec4_09", + "sec5_15", + "sec8_39", + "sec8_41" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_or_empty_section", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-14", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2672, + "total_elapsed_ms": 189928.97, + "total_prompt_tokens": 6987, + "total_retrieved_plane1_tokens": 4863, + "total_tokens": 9659, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 478, + "inference_diagnostic_rules": [ + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1280, + "response_sha256": "05a38aba7d9f7aa8eb8d87db09bd0ca4bf05ffdafc1cf9f6ee182bd3c77cc9e0", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2079, + "response_sha256": "de7bb7c79f3cf80f18e03309e72000ca59d7d583ee553d125f1a2cf21ea2d5d2", + "retrieved_chunk_ids": [ + "sec10_56", + "sec8_39", + "sec8_41" + ], + "retrieved_plane1_tokens": 1210, + "sealed_error_categories": [ + "missing_or_empty_section", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2097, + "response_sha256": "2f48723ecceab85b9717571b5aad7a3167cf5b55ce4c950716000b07c17e767c", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_or_empty_section", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 658, + "inference_diagnostic_rules": [ + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1531, + "response_sha256": "1c8ee1ed44fa62e57dd98ef48a60b4de0692990c796ae92658537cae98992939", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_78", + "sec7_32", + "sec7_34" + ], + "retrieved_plane1_tokens": 1201, + "sealed_error_categories": [ + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-15", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2748, + "total_elapsed_ms": 193717.69, + "total_prompt_tokens": 6871, + "total_retrieved_plane1_tokens": 4843, + "total_tokens": 9619, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1344, + "response_sha256": "62e1b609e8dbb2e736abe1119c96b60e4f7c06aaaeff0d86df37bb8fd263f836", + "retrieved_chunk_ids": [ + "sec7_34", + "sec8_36", + "sec8_39", + "sec9_48" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1994, + "response_sha256": "fadc2f87d8382813f714f94e54ad0779faa7c609ae9e5a9eb729e424fb02368e", + "retrieved_chunk_ids": [ + "sec12_69", + "sec4_09", + "sec7_32", + "sec7_34", + "sec8_39" + ], + "retrieved_plane1_tokens": 1213, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 444, + "inference_diagnostic_rules": [ + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1519, + "response_sha256": "9a93edf94441dd72d42b07467d32db236a80ba28bfd9d211c1104cbab2879e3b", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_72", + "sec12_74", + "sec12_78", + "sec7_32", + "sec7_35" + ], + "retrieved_plane1_tokens": 1189, + "sealed_error_categories": [ + "missing_trust_words", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2014, + "response_sha256": "71d850194cc7802bc306f27071c09a2c3bac9e25de6e52fe39ddad88bb2a95a8", + "retrieved_chunk_ids": [ + "sec4_09", + "sec5_15", + "sec8_39", + "sec8_41" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_or_empty_section", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-16", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 3072, + "total_elapsed_ms": 202319.24, + "total_prompt_tokens": 7042, + "total_retrieved_plane1_tokens": 4832, + "total_tokens": 10114, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1458, + "response_sha256": "b1b58cf1cce946bce54f6ad7885a57457e3bac96683025939d9a66ec96528ca0", + "retrieved_chunk_ids": [ + "sec7_34", + "sec8_36", + "sec8_39", + "sec9_48" + ], + "retrieved_plane1_tokens": 1221, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "missing_expected_tier_label", + "missing_required_boundary_fact", + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2299, + "response_sha256": "32aa411236be7be75d0fc9946b3330fa511b961b9fb0228ffb8e597bc3667311", + "retrieved_chunk_ids": [ + "sec4_09", + "sec6_25", + "sec7_32", + "sec7_34", + "sec8_39" + ], + "retrieved_plane1_tokens": 1211, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "missing_expected_tier_label", + "missing_required_boundary_fact", + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_REQUIRED_JSON_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1658, + "response_sha256": "7a1e10f60a9c0bc2c9de3ea72a3d14c52b97775a7ac204f6ebf8ad276f91545e", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_78", + "sec8_39" + ], + "retrieved_plane1_tokens": 1196, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "wrong_declared_tier", + "missing_verdict_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD", + "INVALID_VERDICT_RECORD" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1627, + "response_sha256": "3f23c1e8779728c85fefd127f466f985928e45827bddbd2a8c2ede40cb4ee617", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1204, + "sealed_error_categories": [ + "missing_trust_words", + "missing_expected_tier_label", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts", + "check_verdicts" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-17", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 500, + "total_elapsed_ms": 39376.39, + "total_prompt_tokens": 1179, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1679, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 500, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1179, + "response_sha256": "9e4821d5f961261f62be544394d03e7ffc60bd4d79db550f17893a6912ecb293", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-18", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 768, + "total_elapsed_ms": 51420.34, + "total_prompt_tokens": 1234, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 2002, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1234, + "response_sha256": "6bb3f2e1289f010e809fc6e84f8f8159de4095f3adbe39624726d9c23a6a83bd", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words", + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-19", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 336, + "total_elapsed_ms": 29045.01, + "total_prompt_tokens": 1182, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1518, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 336, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1182, + "response_sha256": "87fee282c5e20bb815e6d9e9b79d62bd80eadebb784b7792260a7200ba69cba7", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words", + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-20", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 768, + "total_elapsed_ms": 51465.0, + "total_prompt_tokens": 1228, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1996, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1228, + "response_sha256": "6fa4ebbbe5a7e11b3a5ef818ef28d6209df61b5971765077d84ed04fd49108da", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words", + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-21", + "inference_certified_at_turn": 4, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 2738, + "total_elapsed_ms": 90119.84, + "total_prompt_tokens": 6170, + "total_retrieved_plane1_tokens": 4863, + "total_tokens": 8908, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1190, + "response_sha256": "e936e0c16762f0c475bd61bd0da4f1c4318c6aeec9099e9ba79bdfe0fa871fc0", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "missing_required_boundary_fact", + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2047, + "response_sha256": "e936e0c16762f0c475bd61bd0da4f1c4318c6aeec9099e9ba79bdfe0fa871fc0", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "missing_required_boundary_fact", + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 670, + "inference_diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1447, + "response_sha256": "03554bc34a911ff1229162c3361c355025f34440823e1f19853977e33f328f91", + "retrieved_chunk_ids": [ + "sec12_72", + "sec12_74", + "sec12_78", + "sec7_32", + "sec7_34" + ], + "retrieved_plane1_tokens": 1206, + "sealed_error_categories": [ + "missing_or_empty_section", + "missing_or_empty_section", + "f0_gate", + "f0_gate" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 532, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1486, + "response_sha256": "37af7eddc529d0f8d7f784cdfcba4804d3472463634c289c2aab647293208f38", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec12_78", + "sec4_09", + "sec7_35", + "sec8_36" + ], + "retrieved_plane1_tokens": 1205, + "sealed_error_categories": [ + "missing_trust_words", + "f0_gate", + "f0_gate" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-22", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 768, + "total_elapsed_ms": 51447.79, + "total_prompt_tokens": 1241, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 2009, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1241, + "response_sha256": "3dece18c5d5753d72bba2fc9ea784c3190f1dbbe34597546c5ca6fdaca027651", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words", + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-23", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 439, + "total_elapsed_ms": 35542.86, + "total_prompt_tokens": 1176, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1615, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 439, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1176, + "response_sha256": "106081118b440c19112aa35f9d461dbbc6a5de87e398210e3e69672b64b390d6", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words", + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "C", + "id": "HOLD-C-24", + "inference_certified_at_turn": 1, + "oracle_calls": 1, + "stagnation_events": 0, + "total_completion_tokens": 522, + "total_elapsed_ms": 40943.83, + "total_prompt_tokens": 1233, + "total_retrieved_plane1_tokens": 1226, + "total_tokens": 1755, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 522, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1233, + "response_sha256": "0f2d763d8303183ecce0ee58dbd8f0bed928623faa8b97d5c063f79434372c0e", + "retrieved_chunk_ids": [ + "sec6_25", + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1226, + "sealed_error_categories": [ + "missing_trust_words", + "missing_f0_ledger_json_block" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-01", + "inference_certified_at_turn": 4, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1892, + "total_elapsed_ms": 167442.2, + "total_prompt_tokens": 6303, + "total_retrieved_plane1_tokens": 4870, + "total_tokens": 8195, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1204, + "response_sha256": "b577321beb22761f9ab43b3f6e21c3bf16b6254aa2ce490e91b10a2f65addc4b", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 403, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2028, + "response_sha256": "0d1c841499f846ac2ef68293acfb6214c4defb56987ae4ce710b65dbee0187d7", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 322, + "inference_diagnostic_rules": [ + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1474, + "response_sha256": "29ba6d84537d77f7361dfd90f43bdfec50a4bccb4917b186939ba0a727060630", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 399, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1597, + "response_sha256": "4c65a2e7e806d6fb542b117da447051dfce0e5b67155db450de69fe8cc764b21", + "retrieved_chunk_ids": [ + "sec10_58", + "sec4_09", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-02", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 2712, + "total_elapsed_ms": 205799.82, + "total_prompt_tokens": 6208, + "total_retrieved_plane1_tokens": 4866, + "total_tokens": 8920, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1196, + "response_sha256": "c536e232952dc86bdf39b236f5ef5c986372668597521ef1fdf9e011f1a0a1af", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2020, + "response_sha256": "c536e232952dc86bdf39b236f5ef5c986372668597521ef1fdf9e011f1a0a1af", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1468, + "response_sha256": "f714a80d417f973072381795e9391f510112f1b4b9276d5dca921c643c5bded9", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 408, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1524, + "response_sha256": "4a791089f2c714262c3216da26cafebec43a1d4088d2ab0f1b17253573003364", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_69", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1211, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-03", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 210616.97, + "total_prompt_tokens": 6453, + "total_retrieved_plane1_tokens": 4870, + "total_tokens": 9525, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1199, + "response_sha256": "321fcee5a491786dbb964b954caab1cf9139e53471b13dfe67868dfc6121694b", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1511, + "response_sha256": "4ace205c5cb014c4850643a976a13e189287b5a393dd30b7abaa8fbc223ec3f4", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2132, + "response_sha256": "61542309911296c07141596b14cc53bef7073d0d735771f3e5588879717e78b7", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1611, + "response_sha256": "106784edb891c1f6fd981b991ab1134e9cd389106b8554b8ade1ae1c79d68ec6", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-04", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 1315, + "total_elapsed_ms": 112753.44, + "total_prompt_tokens": 6447, + "total_retrieved_plane1_tokens": 4855, + "total_tokens": 7762, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 256, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NO_THEOREM_DECLARED", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1211, + "response_sha256": "021c9caef41adb0ad3c17da9a331ac97595d46836e75acce8b448004b2ba997d", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean_compile_failed", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 302, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NO_THEOREM_DECLARED", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1728, + "response_sha256": "71abea426ae5d93440fb72d195f3573161caaf04883fa607ee776cac063efa18", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 379, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1799, + "response_sha256": "13b4aa6883ee5981f540320a06a14289987631941086d52ddca9dec3f19322d3", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_78", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1213, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 378, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1709, + "response_sha256": "a68a2bbfc823c56e0709482407905eaeb6a8767d1c2cd6798377adb0ceb39293", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-05", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2248, + "total_elapsed_ms": 182663.1, + "total_prompt_tokens": 7042, + "total_retrieved_plane1_tokens": 4855, + "total_tokens": 9290, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 312, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NO_THEOREM_DECLARED", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1210, + "response_sha256": "68da63f20565aa86717edee4b6d95b57a907a6603f86367d52689bfe931e4958", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "lean_compile_failed", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NO_THEOREM_DECLARED", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1783, + "response_sha256": "ed2f79d252cae81641ee1673fb417608f821a9448897929a6c45c36e462e3ef4", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "lean_compile_failed", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 400, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_VACUOUS_TAUTOLOGICAL_HYPOTHESIS", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2266, + "response_sha256": "199808d375c83a76f427e0f14664d807ba9ed07db0c09616cfeda09f7b9716af", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_78", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1213, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_VACUOUS_TAUTOLOGICAL_HYPOTHESIS", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1783, + "response_sha256": "c6231b516017b182fb9a8a8db8ef300652edbc27d9c8489c50c67dd5114e02c0", + "retrieved_chunk_ids": [ + "sec10_58", + "sec4_09", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-06", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 2325, + "total_elapsed_ms": 190516.27, + "total_prompt_tokens": 7135, + "total_retrieved_plane1_tokens": 4862, + "total_tokens": 9460, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1208, + "response_sha256": "f21ef7fb156a1c4441feea15baadfe95453064f15a27772f7574154f8080f373", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 463, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2032, + "response_sha256": "535673d8a7e9696cc113341162d975ca596251f76f4dd90edca3a16cac8b000f", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 561, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1909, + "response_sha256": "4e2bfcbad414aacf9df30dae844a90f573cf3834226cf99bdca7555268f5fae9", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 533, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1986, + "response_sha256": "050f2d974f5acf590de3834a7dfdfb04808013edad84196976a3a397555fa10e", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-07", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 2688, + "total_elapsed_ms": 205245.86, + "total_prompt_tokens": 6300, + "total_retrieved_plane1_tokens": 4866, + "total_tokens": 8988, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1219, + "response_sha256": "52fc0f9d59441ff8a2e2839ff8a437c5c995684accddf12f051a00708d1e61ad", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2044, + "response_sha256": "52fc0f9d59441ff8a2e2839ff8a437c5c995684accddf12f051a00708d1e61ad", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1491, + "response_sha256": "1549858492877b70da5e8806245152a91901256bb72c99795aa76d592d50a63d", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 384, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1546, + "response_sha256": "4a807fa91dad1c500b2489fa9d6702e21d478c8e4a4354f502047a081a485195", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_69", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1211, + "sealed_error_categories": [ + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-08", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 2286, + "total_elapsed_ms": 187818.58, + "total_prompt_tokens": 7837, + "total_retrieved_plane1_tokens": 4846, + "total_tokens": 10123, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1200, + "response_sha256": "b0da175be26d74037dff6157cb7d44a4cf257d8fa2d35d3053db709cfa08a9d5", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_DATA_EIDO_SKIP", + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2025, + "response_sha256": "129ed78458a5ff53bec952dead1f21828f59c5589168e8b4ae734740e761cde5", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "lean_compile_failed", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 372, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NO_THEOREM_DECLARED", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2715, + "response_sha256": "9d6f7a01772bab6a26b2547677e0fb4196fccd3e425ed4b8ef7e2f0a5af25d9f", + "retrieved_chunk_ids": [ + "sec12_78", + "sec13_79", + "sec9_42", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1193, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 378, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1897, + "response_sha256": "8623254002cf5a80d7dc718fc0fb34306302c9850e09669b4439d6b3cd3d09da", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_78", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1213, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-09", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 237090.18, + "total_prompt_tokens": 6270, + "total_retrieved_plane1_tokens": 4866, + "total_tokens": 9342, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1213, + "response_sha256": "a9473cc2d48b5ad6e09c647aeae84f965c2177b429716bafefe7c054b91b2801", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2038, + "response_sha256": "a9473cc2d48b5ad6e09c647aeae84f965c2177b429716bafefe7c054b91b2801", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1481, + "response_sha256": "4943fe5feabdcabb7717f00de83ad9db8677a1c053753b01caf60826b7cb8668", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1538, + "response_sha256": "986fa0b0024521e04d197a3ae523ac51b12127a1fed431675917a7916203c714", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_69", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1211, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": true, + "deployed_pass_at_4": true, + "family": "D", + "id": "HOLD-D-10", + "inference_certified_at_turn": 3, + "oracle_calls": 3, + "stagnation_events": 0, + "total_completion_tokens": 1738, + "total_elapsed_ms": 152302.89, + "total_prompt_tokens": 5189, + "total_retrieved_plane1_tokens": 3653, + "total_tokens": 6927, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": true, + "trajectory_any_pass_at_4": true, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1197, + "response_sha256": "5b4257ec7547b35ed6f5892d09bb77ebbf1041f5c7a0683f98c8716d6a6f740c", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 450, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2022, + "response_sha256": "663561fbd1540793c06637ac0d0754af1eef9314ac29516179c1f64384822750", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 520, + "inference_diagnostic_rules": [], + "inference_tier2_certified": true, + "prompt_tokens": 1970, + "response_sha256": "9b209480a4eaf40bec02be24047f6a8f30df174ebf62e8a221cf0dda1552e534", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_78", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1213, + "sealed_error_categories": [], + "sealed_holdout_passed": true, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + } + ], + "turns_used": 3 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-11", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1935, + "total_elapsed_ms": 161571.7, + "total_prompt_tokens": 7025, + "total_retrieved_plane1_tokens": 4855, + "total_tokens": 8960, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 624, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1221, + "response_sha256": "2017b7ef3dce889e9c9f845fae07065b355db6d9d7f5d8022dad1bdf69b0f307", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 452, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2424, + "response_sha256": "744eceb81b5c7b95cc0c80465d6ee1dc528235e02e5599a1a9bc037fa2d11300", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_78", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1213, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 432, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1802, + "response_sha256": "640e9ba177c5843b1b53280919d908b9e3df3880a2517569450a3571f1c2483f", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 427, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1578, + "response_sha256": "753c3e19c79a53a6fce5b0d04b836491ce735616a4e127dc8aa95eb7bdc04d5e", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-12", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 237139.98, + "total_prompt_tokens": 6231, + "total_retrieved_plane1_tokens": 4866, + "total_tokens": 9303, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1204, + "response_sha256": "fe5741de1121addd911cb686792f05b751400a66d20b267b93893e02305a07a5", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2027, + "response_sha256": "fe5741de1121addd911cb686792f05b751400a66d20b267b93893e02305a07a5", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1472, + "response_sha256": "a7e8173a8c512f92b8d998be3978fa87b94784f44eb33f2779cc8e789544028c", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1528, + "response_sha256": "10c8a41bf76adc67a1d720f4fbd2b2a178b05b8d7e7baffa357eece0f12068c8", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_69", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1211, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-13", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 237154.0, + "total_prompt_tokens": 7532, + "total_retrieved_plane1_tokens": 4875, + "total_tokens": 10604, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1211, + "response_sha256": "d1df54bb04d8d52f65b972745187570311bd1258abf66ad30c61dd644d855def", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_DATA_EIDO_SKIP", + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2034, + "response_sha256": "f9e0c00cf5081a724c2fa5807bc12090a63a29b68bc301ca3321622e6aea6669", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2253, + "response_sha256": "94a02249c7619fecab135c92ef43d2e50595e0e1291983bbff3a70732612c728", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2034, + "response_sha256": "94a02249c7619fecab135c92ef43d2e50595e0e1291983bbff3a70732612c728", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-14", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 2121, + "total_elapsed_ms": 174741.6, + "total_prompt_tokens": 7321, + "total_retrieved_plane1_tokens": 4849, + "total_tokens": 9442, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1207, + "response_sha256": "81a84741343795b08ebd60fa274a4b6e762f0260a9e7eef83fd4ec597e83e177", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_DATA_EIDO_SKIP", + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2032, + "response_sha256": "493dddee5dfc373455a211938248a349dfb38f88c5aa39ea1f03ee936b6c74b2", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 249, + "inference_diagnostic_rules": [ + "MISSING_DATA_EIDO_SKIP", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2450, + "response_sha256": "2c7b89cdfadded277da005c6e2bf446f569dbfd996e8405b10e6995518fd0121", + "retrieved_chunk_ids": [ + "sec12_78", + "sec13_79", + "sec9_42", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1193, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 336, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1632, + "response_sha256": "6fc9526ad7a0e12be0234ea633801b526869bb41bb0d96dfccb9d5e00f605f00", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec12_66", + "sec12_69" + ], + "retrieved_plane1_tokens": 1216, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-15", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 2871, + "total_elapsed_ms": 228530.1, + "total_prompt_tokens": 7754, + "total_retrieved_plane1_tokens": 4868, + "total_tokens": 10625, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NO_THEOREM_DECLARED", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1212, + "response_sha256": "430cb77cd56b85456f951a9ac0c0a9f28d4a57dcedf20ea269fa56b3273d3a4d", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "lean_compile_failed", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NO_THEOREM_DECLARED", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2240, + "response_sha256": "dc0d7357a1304b7aabec4d462ff20ea481bc925f202dda9fb7a6f18cd1587fcf", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "lean_compile_failed", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2265, + "response_sha256": "959765a66ebb1c967ad712969e976e4334a7852a80ca35bc27b9b03f2a8aca17", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_78", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1213, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 567, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2037, + "response_sha256": "931042a59ee3b9e548f12595df12000f21f0647bff159b0a3159473657ccf48f", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-16", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2696, + "total_elapsed_ms": 221340.69, + "total_prompt_tokens": 7348, + "total_retrieved_plane1_tokens": 4853, + "total_tokens": 10044, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 392, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1205, + "response_sha256": "bfd9897e4314f20bff5acba0390c6e92fc12d7b10c113722e626f28b5172fad6", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_DATA_EIDO_SKIP", + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1653, + "response_sha256": "a5c5f3ade6070ace34c0c9e37cfe22d235d9406701d1e2e6cb42dd45e63ff621", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2576, + "response_sha256": "ec5c051d765f3f6975e6259533f84873aa29e528d48ffb0d0d6b32ba152ac788", + "retrieved_chunk_ids": [ + "sec12_78", + "sec13_79", + "sec9_42", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1193, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1914, + "response_sha256": "4d0b46154786db73b8f4c14ce5f172f88602830585b8cea502483ef423008bed", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-17", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 2218, + "total_elapsed_ms": 185440.26, + "total_prompt_tokens": 6296, + "total_retrieved_plane1_tokens": 4870, + "total_tokens": 8514, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1199, + "response_sha256": "66be9777f4695191a5535fe04b4108555e67eef9e9ddccd3dc39897267187893", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1969, + "response_sha256": "090bab0283fcca858f7820cafa0bb39bb21e57ca3b6559d80cfe2476e1e9d390", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 316, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1468, + "response_sha256": "dccfcc7d2287af3aed4483d693d5476d47fa77c689d37ef56e4a3e09bce46dd2", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 366, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1660, + "response_sha256": "3d5d7a61507f7fb09e5b4f2461f534e580032398ebf636c604519bba57f153b3", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-18", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 1603, + "total_elapsed_ms": 141374.45, + "total_prompt_tokens": 6524, + "total_retrieved_plane1_tokens": 4862, + "total_tokens": 8127, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 404, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1216, + "response_sha256": "cb662e7141616990588cfd7985c3e29bc81646570f4904215ed5fe09e9fc41c3", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 393, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1676, + "response_sha256": "6778808a0f34dd29e1d7fe44239fbfd61fa551a82a265a9cba08925f000f364f", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 412, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1817, + "response_sha256": "6700f53d478899b2ffb663c2c68d011d0cc14691ec8ea9468d69956c09fa4c59", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 394, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_FORBIDDEN_META_IMPORT", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1815, + "response_sha256": "0c32453cb46bd16a8bb2f01219b0948a8481c55be84deebbab5263ed4ff20bea", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-19", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 235082.39, + "total_prompt_tokens": 6295, + "total_retrieved_plane1_tokens": 4866, + "total_tokens": 9367, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1219, + "response_sha256": "9e10d21766aef368c1642600d230f137ad8ea6a3c231234b856b2edff7bfe5dd", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2043, + "response_sha256": "08e0872c012f4b9f9d77d7b4022f78611e2783f93fdcc5cafe494177910de5ca", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1490, + "response_sha256": "74ce769ebb38e1d5ee324555febf489b33c662662dcda1a1d807548b7183c27b", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1543, + "response_sha256": "bc00a2ff6f0434a966666bd7043e0b8789f8b5ee9a314e97e8c66a20f26ca8de", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_69", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1211, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-20", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 3, + "total_completion_tokens": 3072, + "total_elapsed_ms": 235133.81, + "total_prompt_tokens": 6215, + "total_retrieved_plane1_tokens": 4866, + "total_tokens": 9287, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1198, + "response_sha256": "f027d2147fbcfe1c725708dc2b1abc344913130b40223776aa754b4c09d69fc8", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2023, + "response_sha256": "f027d2147fbcfe1c725708dc2b1abc344913130b40223776aa754b4c09d69fc8", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1468, + "response_sha256": "827db39256da9d4196e054eba89e6677ac52465d6a3eb245c00d65c8f1ce41aa", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1526, + "response_sha256": "56caf063f94cd97804afd4ccfc8ec6692b0f2b66cef2612014225838d5cb89ed", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_69", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1211, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-21", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 1686, + "total_elapsed_ms": 146079.8, + "total_prompt_tokens": 6803, + "total_retrieved_plane1_tokens": 4849, + "total_tokens": 8489, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 436, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NO_THEOREM_DECLARED", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1222, + "response_sha256": "7f8ae54df57e7989b43119d04010ad9d2eb10ecd9b9d64469ddb3313c9b28908", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 422, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NO_THEOREM_DECLARED", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1955, + "response_sha256": "7c0a344db0227dde6590e7f8ba9f1c5616cf445d2e0acd3767a5b3c6817316de", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 414, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NO_THEOREM_DECLARED", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1863, + "response_sha256": "8b5df4cc826977c1e95d6d99f0896016b1c95b3c4d1a196b280c7802be3dfdfd", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 414, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NO_THEOREM_DECLARED", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1763, + "response_sha256": "8b5df4cc826977c1e95d6d99f0896016b1c95b3c4d1a196b280c7802be3dfdfd", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean_compile_failed" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-22", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 1, + "total_completion_tokens": 3072, + "total_elapsed_ms": 234511.52, + "total_prompt_tokens": 7713, + "total_retrieved_plane1_tokens": 4868, + "total_tokens": 10785, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NO_THEOREM_DECLARED", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_CLAIM_TIER_LABEL", + "MISSING_PROMPT_COMMIT_SHA40" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1202, + "response_sha256": "d94f3288cc25881d1af3a362f74a78555ebdaf16f39b4b92e6675c2186214a5c", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "corrupted_or_truncated_40char_sha", + "lean_compile_failed", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_NO_THEOREM_DECLARED", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2231, + "response_sha256": "50eb4ba895c3f63aadfb20309ed1b867f1f7588998eb236168e5f0a7adaf10e0", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "lean_compile_failed", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2255, + "response_sha256": "fdc6635308b1abfe9b0d361ed96848b5ec4dcbc1cf083303390832695d17c85f", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_78", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1213, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2025, + "response_sha256": "ff4325cd5507b99bdee6992a27b6eee110c90747132653f19a8e543585bd423f", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-23", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 0, + "total_completion_tokens": 1938, + "total_elapsed_ms": 166904.79, + "total_prompt_tokens": 7589, + "total_retrieved_plane1_tokens": 4857, + "total_tokens": 9527, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 455, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "MISSING_40CHAR_VERDICT_SHA", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1215, + "response_sha256": "c9b281375d86ddedb6fa1b208a4a8b124e0e1ba657d95a0cb9a11e4be156e276", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "corrupted_or_truncated_40char_sha", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 493, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2097, + "response_sha256": "ef86e14a0de1def58a292942c53bc1dfc3b7ddd47071e01e1dfcbd3a293de642", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 497, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "EMBEDDED_LEAN_NON_INIT_IMPORT", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2147, + "response_sha256": "615310dead2384387541583c92d50a15169f73bb549aae7ee4a2aa26e9fb733f", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec7_32" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "lean4_auditor", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "completion_tokens": 493, + "inference_diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_FORBIDDEN_META_IMPORT", + "EMBEDDED_LEAN_NO_THEOREM_DECLARED", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_FORBIDDEN_META_IMPORT", + "EMBEDDED_LEAN_NO_THEOREM_DECLARED" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2130, + "response_sha256": "95d19930e17e38c20d7fc57074d13583aba50d5cdbc0d5fd0d51fb32f5b9b3fc", + "retrieved_chunk_ids": [ + "sec10_58", + "sec12_66", + "sec9_42" + ], + "retrieved_plane1_tokens": 1207, + "sealed_error_categories": [ + "lean4_auditor", + "lean4_auditor", + "mde_guard" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "deployed_pass_at_1": false, + "deployed_pass_at_2": false, + "deployed_pass_at_3": false, + "deployed_pass_at_4": false, + "family": "D", + "id": "HOLD-D-24", + "inference_certified_at_turn": null, + "oracle_calls": 4, + "stagnation_events": 2, + "total_completion_tokens": 2643, + "total_elapsed_ms": 217698.93, + "total_prompt_tokens": 6261, + "total_retrieved_plane1_tokens": 4866, + "total_tokens": 8904, + "trajectory_any_pass_at_1": false, + "trajectory_any_pass_at_2": false, + "trajectory_any_pass_at_3": false, + "trajectory_any_pass_at_4": false, + "turns": [ + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1211, + "response_sha256": "efa7245f4e85ea765d6939337da2164ddbcdd0ebd11cc8b95e61b02581de4b98", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 2035, + "response_sha256": "f53a1c54d7e02d24292751aa26a777ac56814fc56177f1ccf76fab2679540ea7", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec11_61", + "sec4_09" + ], + "retrieved_plane1_tokens": 1220, + "sealed_error_categories": [ + "missing_data_eido_skip_container", + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "completion_tokens": 768, + "inference_diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1479, + "response_sha256": "75a0dfd8acc88d52e8e9006f139edfb8ff355ccf687452e508be674931db4116", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_65", + "sec12_78" + ], + "retrieved_plane1_tokens": 1215, + "sealed_error_categories": [ + "missing_fenced_lean_block", + "missing_or_empty_section", + "missing_or_empty_section" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "completion_tokens": 339, + "inference_diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "inference_tier2_certified": false, + "prompt_tokens": 1536, + "response_sha256": "f8e9b6d0b6bafee3aa7d05c2e264905f57cc02cae392122fa99d03f030d6d3a3", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_69", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1211, + "sealed_error_categories": [ + "lean_compile_failed", + "missing_trust_words" + ], + "sealed_holdout_passed": false, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + } + ], + "pass_curve": { + "k=1": { + "pass_rate_pct": 0.0, + "passed": 0, + "total": 96 + }, + "k=2": { + "pass_rate_pct": 0.0, + "passed": 0, + "total": 96 + }, + "k=3": { + "pass_rate_pct": 1.04, + "passed": 1, + "total": 96 + }, + "k=4": { + "pass_rate_pct": 1.04, + "passed": 1, + "total": 96 + } + }, + "total_items": 96, + "trajectory_any_pass_curve": { + "k=1": { + "pass_rate_pct": 0.0, + "passed": 0, + "total": 96 + }, + "k=2": { + "pass_rate_pct": 0.0, + "passed": 0, + "total": 96 + }, + "k=3": { + "pass_rate_pct": 1.04, + "passed": 1, + "total": 96 + }, + "k=4": { + "pass_rate_pct": 1.04, + "passed": 1, + "total": 96 + } + } + } + }, + "evaluation_protocol": { + "families": { + "A": 24, + "B": 24, + "C": 24, + "D": 24 + }, + "hidden_holdout_fixtures_exposed_to_model": false, + "inference_oracle_mode": "strict_label_free_tier2_compiler_and_auditor", + "max_predict_tokens": 768, + "max_turns": 4, + "retrieval_max_tokens": 1250, + "seed": 42, + "single_pass_wave1_baseline": { + "B0_k1_no_retrieval": "0/96", + "B1_32216_tok_full_spec": "0/96", + "P2B_k1_no_retrieval": "0/96" + }, + "split": "private_holdout.jsonl", + "split_sha256": "4f5eac7fea63b5495896c23820180668d1e4d0a0a0a14297c63bf93419d989e3", + "temperature": 0.0, + "top_p": 1.0, + "total_items": 96 + }, + "issue_ref": "Proven-Commons/mcproof#5", + "privacy_verification": { + "head_middle_tail_40char_scan_clean": true, + "local_path_and_ip_scrub_clean": true, + "normalized_8gram_scan_clean": true + }, + "provenance": { + "base_model_filename": "Qwen3.5-4B-Q8_0.gguf", + "base_model_sha256": "10cc391b403021dd11c614679d2fd92f611c3681d29e29651b717316965d61e1", + "mcproof_v02_commit": "155a474b5270a7771017a62f8e1db71eb2b75fe5", + "p2b_adapter_filename": "formal-verification.p2b.f16.gguf", + "p2b_adapter_sha256": "728272c0974b795ceb990fde1e215b62d64e6faf04fb799c0fd3e01fa4539ba5", + "plane1_formal_spec_sha256": "49beff2a0185cfa4032170b614a543c0d67541494e858803d21cfb837111b230", + "runtime": { + "chat_template_enable_thinking": false, + "llama_server_version": "0.00.000.154 I srv llama_server: initializing ...", + "parallel_slots": 8, + "slot0_n_ctx": 8192, + "total_n_ctx_reported": 8192 + } + }, + "schema": "mcproof-sealed-holdout-repair-receipt/v0.2" +} diff --git a/compile/artifacts/evaluation/repair_loop/validation_repair_b0.json b/compile/artifacts/evaluation/repair_loop/validation_repair_b0.json new file mode 100644 index 0000000..437a340 --- /dev/null +++ b/compile/artifacts/evaluation/repair_loop/validation_repair_b0.json @@ -0,0 +1,5788 @@ +{ + "arm": "B0", + "budget_totals": { + "completion_tokens": 98510, + "inference_certified_items": 23, + "mean_retrieved_plane1_tokens_per_turn": 1194.8, + "oracle_calls": 139, + "prompt_tokens": 241755, + "replies_used": 139, + "stagnation_events": 38, + "total_tokens": 340265 + }, + "by_family": { + "A": { + "k=1": { + "passed": 1, + "total": 12 + }, + "k=2": { + "passed": 1, + "total": 12 + }, + "k=3": { + "passed": 1, + "total": 12 + }, + "k=4": { + "passed": 1, + "total": 12 + } + }, + "B": { + "k=1": { + "passed": 4, + "total": 12 + }, + "k=2": { + "passed": 5, + "total": 12 + }, + "k=3": { + "passed": 5, + "total": 12 + }, + "k=4": { + "passed": 5, + "total": 12 + } + }, + "C": { + "k=1": { + "passed": 0, + "total": 12 + }, + "k=2": { + "passed": 0, + "total": 12 + }, + "k=3": { + "passed": 0, + "total": 12 + }, + "k=4": { + "passed": 0, + "total": 12 + } + }, + "D": { + "k=1": { + "passed": 0, + "total": 12 + }, + "k=2": { + "passed": 0, + "total": 12 + }, + "k=3": { + "passed": 0, + "total": 12 + }, + "k=4": { + "passed": 2, + "total": 12 + } + } + }, + "include_plane1_retrieval": true, + "items": [ + { + "family": "A", + "id": "PUB-A-V01", + "inference_certified": true, + "inference_certified_at_turn": 1, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 1, + "pass_at_1": true, + "pass_at_2": true, + "pass_at_3": true, + "pass_at_4": true, + "passed": true, + "refuted_routes": [], + "solved_at_turn": 1, + "stagnation_events": 0, + "total_completion_tokens": 174, + "total_elapsed_ms": 27537.95, + "total_prompt_tokens": 1227, + "total_retrieved_plane1_tokens": 1206, + "total_tokens": 1401, + "turns": [ + { + "candidate_state_hash": "33aa02661cc94cc0221f1aa4", + "completion_tokens": 174, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 6454.89, + "eval_passed": true, + "failure_signature": "PASS", + "generation_ms": 20383.07, + "inference_oracle_ms": 7154.88, + "inference_passed": true, + "passed": true, + "prompt_tokens": 1227, + "response_sha256": "78c2d98798773ac4b682d24a23802f74ff37c2e2c5ba4f656ee6499f43dbcb08", + "retrieved_chunk_ids": [ + "sec9_43", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "family": "A", + "id": "PUB-A-V02", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "4dcf571b87de", + "turn": 1 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "471b0d6164c3", + "turn": 2 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "0bffbbf6293a", + "turn": 3 + }, + { + "primary_rule": "FORBIDDEN_META_IMPORT", + "state_hash": "86d805a33ab8", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 4096, + "total_elapsed_ms": 417667.54, + "total_prompt_tokens": 8172, + "total_retrieved_plane1_tokens": 4797, + "total_tokens": 12268, + "turns": [ + { + "candidate_state_hash": "4dcf571b87de96d773e85179", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 344.3, + "eval_passed": false, + "failure_signature": "7bf11c12f10531dbb6458573", + "generation_ms": 83354.11, + "inference_oracle_ms": 354.04, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1394, + "response_sha256": "15adad9ee7d45db1263c6cae6fe21eed4bc79d504e185415302dbf0dbe0d642e", + "retrieved_chunk_ids": [ + "sec4_09", + "sec4_12", + "sec9_43", + "sec9_44" + ], + "retrieved_plane1_tokens": 1194, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "471b0d6164c3ad639a5a7a5b", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 356.37, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 86635.0, + "inference_oracle_ms": 363.56, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2421, + "response_sha256": "08f0d4e0cf9499aefce4ceb3a481a5783c484c459709e62183eb382ce52b9757", + "retrieved_chunk_ids": [ + "sec9_43", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "0bffbbf6293a6dc7581a89dd", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 548.41, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 125472.93, + "inference_oracle_ms": 1104.38, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2541, + "response_sha256": "3d27e68ca56386d3743ca2cf96ddb8d2614919aa9c40b15f6571072954a71036", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1199, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "86d805a33ab8ce5cde451588", + "completion_tokens": 1024, + "diagnostic_count": 4, + "diagnostic_rules": [ + "FORBIDDEN_META_IMPORT", + "FORBIDDEN_META_IMPORT", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 421.28, + "eval_passed": false, + "failure_signature": "11cfb398fb4126b17586fca0", + "generation_ms": 119970.13, + "inference_oracle_ms": 413.39, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1816, + "response_sha256": "e69ad5d6a7e3f36eb9dba85ae7a0b2fabacd1cfe923e0d703721959307ab1a00", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_67", + "sec12_69", + "sec12_72", + "sec12_78" + ], + "retrieved_plane1_tokens": 1198, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V03", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "b1b1b0d5968f", + "turn": 1 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "9633be060443", + "turn": 2 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "f7a3524483bd", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "d039ebf4e441", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 2511, + "total_elapsed_ms": 237667.28, + "total_prompt_tokens": 6697, + "total_retrieved_plane1_tokens": 4750, + "total_tokens": 9208, + "turns": [ + { + "candidate_state_hash": "b1b1b0d5968f966793db3c2f", + "completion_tokens": 321, + "diagnostic_count": 6, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 357.25, + "eval_passed": false, + "failure_signature": "55b3d5fcf2f8e6de070139e8", + "generation_ms": 29688.37, + "inference_oracle_ms": 373.23, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1145, + "response_sha256": "ddb483c582a7d0ee7423935a120d8c67917cb4bf5cd18925d1ebad76fa1deef1", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec4_09", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1159, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "9633be06044348846ed5c8b5", + "completion_tokens": 377, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 373.19, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 25783.45, + "inference_oracle_ms": 365.34, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1725, + "response_sha256": "7ea7c63c70f4565db666bb35ee1ccbc25457351d518b3c6f28f5aef067ec274c", + "retrieved_chunk_ids": [ + "sec12_65", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "f7a3524483bdb34007e53a55", + "completion_tokens": 789, + "diagnostic_count": 5, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 333.75, + "eval_passed": false, + "failure_signature": "6b584f9ee287d28264c80f34", + "generation_ms": 64107.87, + "inference_oracle_ms": 349.57, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1654, + "response_sha256": "dc4d4beb8f7f0f3120bd50ec06706539871bae75c1de055c8b5370e3fc64aec7", + "retrieved_chunk_ids": [ + "sec12_65", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "d039ebf4e44113f6dfd1626a", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 343.22, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 116650.45, + "inference_oracle_ms": 349.0, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2173, + "response_sha256": "c713fd0a0e5e6c5e346d340afe06df022bbf6a90fda85dd7d200daa2af288902", + "retrieved_chunk_ids": [ + "sec12_65", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V04", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "45908037ecd5", + "turn": 1 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "722eb4f5c733", + "turn": 2 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "3bea782470bd", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "a7b43342d935", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 2, + "total_completion_tokens": 3276, + "total_elapsed_ms": 336005.56, + "total_prompt_tokens": 6629, + "total_retrieved_plane1_tokens": 4807, + "total_tokens": 9905, + "turns": [ + { + "candidate_state_hash": "45908037ecd565f0d0058fcf", + "completion_tokens": 204, + "diagnostic_count": 2, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 353.97, + "eval_passed": false, + "failure_signature": "a40fc511886323c0c3b1beeb", + "generation_ms": 22044.51, + "inference_oracle_ms": 348.83, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1231, + "response_sha256": "29fcfc42e73aebd7bb867d2755fce158e1179d1bb73613eda3bdf6514a16dc04", + "retrieved_chunk_ids": [ + "sec9_43", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "722eb4f5c733d2b0a7f478d4", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 371.86, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 82121.15, + "inference_oracle_ms": 371.41, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1478, + "response_sha256": "e05a5c95425bf69e42f1221fe07c458193614db4e4df0aa7a1758da2270c57b3", + "retrieved_chunk_ids": [ + "sec12_65", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "3bea782470bd08d8fe216b19", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 562.5, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 103148.91, + "inference_oracle_ms": 514.9, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2303, + "response_sha256": "506bf6de3a7436a555ddc76bc7a9e573168859ff2da73fdb18610d48cf97f96e", + "retrieved_chunk_ids": [ + "sec9_43", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "a7b43342d935871806650924", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 373.57, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 127058.56, + "inference_oracle_ms": 397.29, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1617, + "response_sha256": "d9558e12cca8c2045f73e5579dbd34fde092d32475b048c3ef4dc3cc596d90b6", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_67", + "sec12_69", + "sec12_72", + "sec12_78" + ], + "retrieved_plane1_tokens": 1198, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V05", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "NON_INIT_IMPORT", + "state_hash": "83e8ff756b52", + "turn": 1 + }, + { + "primary_rule": "FORBIDDEN_META_IMPORT", + "state_hash": "911815c9e403", + "turn": 2 + }, + { + "primary_rule": "FORBIDDEN_META_IMPORT", + "state_hash": "b7c30b254040", + "turn": 3 + }, + { + "primary_rule": "UNPROVED_SORRY", + "state_hash": "083d0c42cd1e", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 3721, + "total_elapsed_ms": 386672.36, + "total_prompt_tokens": 7420, + "total_retrieved_plane1_tokens": 4761, + "total_tokens": 11141, + "turns": [ + { + "candidate_state_hash": "83e8ff756b522536b7e4dd7d", + "completion_tokens": 1024, + "diagnostic_count": 4, + "diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 352.41, + "eval_passed": false, + "failure_signature": "0007fb1cf8af203006ed39cf", + "generation_ms": 84190.86, + "inference_oracle_ms": 341.94, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1294, + "response_sha256": "1b889ed766d056b3ae40065681c61f3d66ef2ade9cf45e79cd0e16a5baf24455", + "retrieved_chunk_ids": [ + "sec12_78", + "sec4_09", + "sec4_10", + "sec9_42", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "911815c9e40361b7a407fe7e", + "completion_tokens": 1024, + "diagnostic_count": 3, + "diagnostic_rules": [ + "FORBIDDEN_META_IMPORT", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 359.41, + "eval_passed": false, + "failure_signature": "d8849c899d9d7633e34291ff", + "generation_ms": 88765.24, + "inference_oracle_ms": 360.07, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2177, + "response_sha256": "9e8be5b179481702d7e7a1e3386cf5e798fd3d8812cfd7a16596481d407700e8", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1199, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "b7c30b254040f08d7aec560c", + "completion_tokens": 1024, + "diagnostic_count": 3, + "diagnostic_rules": [ + "FORBIDDEN_META_IMPORT", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 352.56, + "eval_passed": false, + "failure_signature": "d8849c899d9d7633e34291ff", + "generation_ms": 125578.1, + "inference_oracle_ms": 1297.7, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2140, + "response_sha256": "6ab6eecff4cf5cc9098e708434c67d07a92af18e41fd044742308c074adc3d97", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1199, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "083d0c42cd1e75e38e76a8fd", + "completion_tokens": 649, + "diagnostic_count": 5, + "diagnostic_rules": [ + "UNPROVED_SORRY", + "UNPROVED_SORRY", + "UNPROVED_SORRY", + "FORBIDDEN_META_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 1189.24, + "eval_passed": false, + "failure_signature": "d9fd88b6258ca9b0ccbd927b", + "generation_ms": 82160.78, + "inference_oracle_ms": 3977.67, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1809, + "response_sha256": "957f69cafc4008355152addc39aac0601e1f142a04c819725b522f3c17761493", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_66", + "sec12_67", + "sec12_70" + ], + "retrieved_plane1_tokens": 1181, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V06", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "ec955e469e02", + "turn": 1 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "c8df98d46b30", + "turn": 2 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "767d1b48760b", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "179b803f0e58", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 1014, + "total_elapsed_ms": 91910.37, + "total_prompt_tokens": 5755, + "total_retrieved_plane1_tokens": 4743, + "total_tokens": 6769, + "turns": [ + { + "candidate_state_hash": "ec955e469e02aa3740996642", + "completion_tokens": 130, + "diagnostic_count": 2, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 383.4, + "eval_passed": false, + "failure_signature": "31a701ab183f2d70d376d4ef", + "generation_ms": 16555.14, + "inference_oracle_ms": 387.14, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1146, + "response_sha256": "65b21db8d4bc82592ec4d24cb062942864c892decbbf5d622bda49c0c61cab5f", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec4_09", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1174, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "c8df98d46b3095995aa1a9ee", + "completion_tokens": 172, + "diagnostic_count": 2, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 390.56, + "eval_passed": false, + "failure_signature": "9ea215c56f627abc041494bb", + "generation_ms": 12161.94, + "inference_oracle_ms": 397.44, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1391, + "response_sha256": "b8f6471d4c3774f85f3dbb6bc2f88e455d7b35c13e3f06ba99c5336499de4a85", + "retrieved_chunk_ids": [ + "sec12_65", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "767d1b48760beacf1df2973e", + "completion_tokens": 523, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 409.17, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 38917.8, + "inference_oracle_ms": 413.91, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1431, + "response_sha256": "433b92b1770576d4a1f8fd002198d0162fe59be73a5bd6197cf403e558fd3c0b", + "retrieved_chunk_ids": [ + "sec12_65", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "179b803f0e58641ab36cce21", + "completion_tokens": 189, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 378.59, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 22704.13, + "inference_oracle_ms": 372.87, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1787, + "response_sha256": "c62616e885bf3828943482a1d621a24e5f1722a005db31cf3a51bc355a9e9def", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1175, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V07", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "2cd3b07537f4", + "turn": 1 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "91018bfc4a43", + "turn": 2 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "cb3a23aea9cb", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "966866998112", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 4078, + "total_elapsed_ms": 420312.26, + "total_prompt_tokens": 8176, + "total_retrieved_plane1_tokens": 4699, + "total_tokens": 12254, + "turns": [ + { + "candidate_state_hash": "2cd3b07537f41933cd63bf4e", + "completion_tokens": 1006, + "diagnostic_count": 9, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC" + ], + "eval_grader_ms": 333.64, + "eval_passed": false, + "failure_signature": "1263b09ef9641f604caa5d6c", + "generation_ms": 79860.16, + "inference_oracle_ms": 342.57, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1222, + "response_sha256": "ca0cddd005b66cddf6566f650c5b67869606f8feb57959025ea304ee4a80ce8c", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec4_09", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1174, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "91018bfc4a438bb951804ae2", + "completion_tokens": 1024, + "diagnostic_count": 8, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 374.19, + "eval_passed": false, + "failure_signature": "2085cb1f7f1d54d3e5919664", + "generation_ms": 91029.17, + "inference_oracle_ms": 377.18, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2316, + "response_sha256": "7b976ce9a51fbbadc43894522fc986a7e063362ccc743dde94b1751d03fb6981", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1175, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "cb3a23aea9cb7fbfe5389699", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 1273.72, + "eval_passed": false, + "failure_signature": "f29affbad2a4bdd718725b07", + "generation_ms": 127799.86, + "inference_oracle_ms": 378.76, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2374, + "response_sha256": "797dcbd771696e3cfc7f39681220c496ce202a589bb92279734707e052641a00", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1175, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "96686699811224d5aceea932", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 413.5, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 120099.75, + "inference_oracle_ms": 424.81, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2264, + "response_sha256": "233831bd6111c83d91cabc35272ce63962e6339a9513bba2764d225f57cb927f", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1175, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V08", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "ecb9b98db263", + "turn": 1 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "21cd940a85c4", + "turn": 2 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "21cd940a85c4", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "bf58b4c47595", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 3982, + "total_elapsed_ms": 410944.43, + "total_prompt_tokens": 7935, + "total_retrieved_plane1_tokens": 4778, + "total_tokens": 11917, + "turns": [ + { + "candidate_state_hash": "ecb9b98db26338c6b57833f3", + "completion_tokens": 1024, + "diagnostic_count": 5, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 357.97, + "eval_passed": false, + "failure_signature": "757029baa073ae042092dfbe", + "generation_ms": 83552.54, + "inference_oracle_ms": 371.96, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1316, + "response_sha256": "cb2e2d3a2dbeeec0784c42c451bdce49dd1ce0cfcc0f38742baca2147b064bb8", + "retrieved_chunk_ids": [ + "sec4_09", + "sec4_12", + "sec9_43", + "sec9_44" + ], + "retrieved_plane1_tokens": 1194, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "21cd940a85c45a225c419a4a", + "completion_tokens": 967, + "diagnostic_count": 5, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 390.4, + "eval_passed": false, + "failure_signature": "60780edc79fe5cbce18b2d4d", + "generation_ms": 80603.91, + "inference_oracle_ms": 377.58, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2378, + "response_sha256": "355ce636dd2244995570b723c98b9d89a0c1cea22f5f82fb0b3174ad9d9d06aa", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1199, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "21cd940a85c45a225c419a4a", + "completion_tokens": 967, + "diagnostic_count": 5, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 370.36, + "eval_passed": false, + "failure_signature": "60780edc79fe5cbce18b2d4d", + "generation_ms": 120980.48, + "inference_oracle_ms": 390.63, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2422, + "response_sha256": "355ce636dd2244995570b723c98b9d89a0c1cea22f5f82fb0b3174ad9d9d06aa", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1199, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "bf58b4c475959834a79a3716", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 400.86, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 124252.84, + "inference_oracle_ms": 414.49, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1819, + "response_sha256": "7e549c6ae247f79609325e6e31179acc0055131d111b331f5d88eb0f8fb02c1a", + "retrieved_chunk_ids": [ + "sec12_67", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1186, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V09", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "81cb54245143", + "turn": 1 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "81cb54245143", + "turn": 2 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "c51710a34c36", + "turn": 3 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "904342db8f6e", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 1403, + "total_elapsed_ms": 110700.87, + "total_prompt_tokens": 6444, + "total_retrieved_plane1_tokens": 4782, + "total_tokens": 7847, + "turns": [ + { + "candidate_state_hash": "81cb542451430b208308fba7", + "completion_tokens": 231, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 405.13, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 14277.46, + "inference_oracle_ms": 414.68, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1312, + "response_sha256": "37d4e81f62934c47650d1c55badb4d9cdc7aa23a0802610ad898d2ec81e6c059", + "retrieved_chunk_ids": [ + "sec9_43", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "81cb542451430b208308fba7", + "completion_tokens": 231, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 397.36, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 16400.32, + "inference_oracle_ms": 423.86, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1539, + "response_sha256": "37d4e81f62934c47650d1c55badb4d9cdc7aa23a0802610ad898d2ec81e6c059", + "retrieved_chunk_ids": [ + "sec12_65", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "c51710a34c363db487582863", + "completion_tokens": 704, + "diagnostic_count": 3, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 351.8, + "eval_passed": false, + "failure_signature": "fc4f32afbd66322f1f4665df", + "generation_ms": 60105.63, + "inference_oracle_ms": 358.12, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1554, + "response_sha256": "1b3f25b32b3f74b1b39e1b3acfb2d3bbc5efd62e78182d6b3f38b9673d66c72b", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_76", + "sec12_78" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "904342db8f6e9e1defb4a1dc", + "completion_tokens": 237, + "diagnostic_count": 3, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 451.56, + "eval_passed": false, + "failure_signature": "80c84b3cfbc23be7cab385e2", + "generation_ms": 18325.6, + "inference_oracle_ms": 395.2, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2039, + "response_sha256": "751983722706e71668fd29c5cef1b0468404f30343a4e545e8f62aa06de60c0c", + "retrieved_chunk_ids": [ + "sec12_65", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V10", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "NON_INIT_IMPORT", + "state_hash": "9528ea2ed26c", + "turn": 1 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "1babd81033a6", + "turn": 2 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "1babd81033a6", + "turn": 3 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "72aeda0ed8aa", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 3279, + "total_elapsed_ms": 235562.64, + "total_prompt_tokens": 7303, + "total_retrieved_plane1_tokens": 4805, + "total_tokens": 10582, + "turns": [ + { + "candidate_state_hash": "9528ea2ed26c9387ec52c94f", + "completion_tokens": 771, + "diagnostic_count": 4, + "diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "NON_INIT_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 246.34, + "eval_passed": false, + "failure_signature": "dfcb1948468148dd002cc5c0", + "generation_ms": 55616.59, + "inference_oracle_ms": 243.97, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1304, + "response_sha256": "f56f845dd49fe220954afaa1ec2c25cd65f4925f31492fad6fd7763ffc4cafe7", + "retrieved_chunk_ids": [ + "sec4_09", + "sec9_43", + "sec9_53" + ], + "retrieved_plane1_tokens": 1210, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "1babd81033a635d1226c12b4", + "completion_tokens": 742, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 437.3, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 56867.15, + "inference_oracle_ms": 461.34, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2281, + "response_sha256": "692cb3c4d8e40acf5fd29b8bdab01bcc4962e53536b501869d0ed547c7c35764", + "retrieved_chunk_ids": [ + "sec12_66", + "sec9_43", + "sec9_53" + ], + "retrieved_plane1_tokens": 1210, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "1babd81033a635d1226c12b4", + "completion_tokens": 742, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 381.83, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 51536.59, + "inference_oracle_ms": 395.78, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2161, + "response_sha256": "692cb3c4d8e40acf5fd29b8bdab01bcc4962e53536b501869d0ed547c7c35764", + "retrieved_chunk_ids": [ + "sec12_66", + "sec9_43", + "sec9_53" + ], + "retrieved_plane1_tokens": 1210, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "72aeda0ed8aabea9c4d1ca47", + "completion_tokens": 1024, + "diagnostic_count": 3, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 366.45, + "eval_passed": false, + "failure_signature": "49a1e63c3e1b96cb2e8af376", + "generation_ms": 70078.91, + "inference_oracle_ms": 362.31, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1557, + "response_sha256": "c91062eef683e456c37202654660460856bf05ff2f209899490155c3d485fbed", + "retrieved_chunk_ids": [ + "sec12_69", + "sec12_70", + "sec12_72", + "sec12_74", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1175, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V11", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "NO_THEOREM_DECLARED", + "state_hash": "8ed238610e62", + "turn": 1 + }, + { + "primary_rule": "NO_THEOREM_DECLARED", + "state_hash": "bdfb5999618c", + "turn": 2 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "f58a855e28e5", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "b038da1b7516", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 4096, + "total_elapsed_ms": 367401.47, + "total_prompt_tokens": 7095, + "total_retrieved_plane1_tokens": 4816, + "total_tokens": 11191, + "turns": [ + { + "candidate_state_hash": "8ed238610e62789d7feb9a92", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 370.83, + "eval_passed": false, + "failure_signature": "53f11506339e1345695bc3a2", + "generation_ms": 124835.27, + "inference_oracle_ms": 368.08, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1304, + "response_sha256": "62275120f910b0f704d45f7dd96da8bd17ad491de8e6ce8b2ff8608806639e84", + "retrieved_chunk_ids": [ + "sec4_09", + "sec9_43", + "sec9_53" + ], + "retrieved_plane1_tokens": 1210, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "bdfb5999618ca79cd1aa9b72", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 345.53, + "eval_passed": false, + "failure_signature": "53f11506339e1345695bc3a2", + "generation_ms": 91983.11, + "inference_oracle_ms": 351.22, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2087, + "response_sha256": "75aeb16d6e92a5946241886a09b835e86d22fcf850ec6d2df755a693939240ab", + "retrieved_chunk_ids": [ + "sec12_66", + "sec9_43", + "sec9_53" + ], + "retrieved_plane1_tokens": 1210, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "f58a855e28e5a425300adbc0", + "completion_tokens": 1024, + "diagnostic_count": 3, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 432.23, + "eval_passed": false, + "failure_signature": "aaa66cefa82e0dd288e1b7e6", + "generation_ms": 76736.58, + "inference_oracle_ms": 432.2, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1692, + "response_sha256": "a9fc3fe505ad3b76e8e2aa7f8fe1190ac163236a8c017daecad0cd710a06f980", + "retrieved_chunk_ids": [ + "sec12_67", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1186, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "b038da1b7516a68b75dd8d56", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 345.76, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 72354.46, + "inference_oracle_ms": 340.55, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2012, + "response_sha256": "a81fc85698f83d44c8b4b559ac828f42116cb8bef73420a62dc7c5c094b349f5", + "retrieved_chunk_ids": [ + "sec12_66", + "sec9_43", + "sec9_53" + ], + "retrieved_plane1_tokens": 1210, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V12", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "fdc626f56cf3", + "turn": 1 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "fdc626f56cf3", + "turn": 2 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "5b682f6c8866", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "06a43a71812e", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 2, + "total_completion_tokens": 2509, + "total_elapsed_ms": 217186.5, + "total_prompt_tokens": 6258, + "total_retrieved_plane1_tokens": 4774, + "total_tokens": 8767, + "turns": [ + { + "candidate_state_hash": "fdc626f56cf3d26631097fee", + "completion_tokens": 339, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 368.32, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 39239.55, + "inference_oracle_ms": 377.46, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1240, + "response_sha256": "60a57d57e6b71526f3a9310c1c8ead7629e6199897e26d30736914eb06c4ab46", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1199, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "fdc626f56cf3d26631097fee", + "completion_tokens": 339, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 393.32, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 35279.79, + "inference_oracle_ms": 430.61, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1701, + "response_sha256": "60a57d57e6b71526f3a9310c1c8ead7629e6199897e26d30736914eb06c4ab46", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1199, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "5b682f6c88668e26ab7a2ce8", + "completion_tokens": 807, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 372.09, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 58854.52, + "inference_oracle_ms": 361.2, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1470, + "response_sha256": "c659df6689650aa6da0a4ceeefec8969f43584e3e4ef3a0422059fcdad41e042", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_70", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1170, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "06a43a71812e50e5b35b6d24", + "completion_tokens": 1024, + "diagnostic_count": 3, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 350.46, + "eval_passed": false, + "failure_signature": "7bf11c12f10531dbb6458573", + "generation_ms": 82285.77, + "inference_oracle_ms": 357.6, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1847, + "response_sha256": "83347722aa7152c238e83aa7ce11a667c7b21065a19b31bb9ec2ab2007aa9961", + "retrieved_chunk_ids": [ + "sec12_67", + "sec12_74", + "sec12_76", + "sec9_48" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "B", + "id": "PUB-B-V01", + "inference_certified": true, + "inference_certified_at_turn": 1, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 1, + "pass_at_1": true, + "pass_at_2": true, + "pass_at_3": true, + "pass_at_4": true, + "passed": true, + "refuted_routes": [], + "solved_at_turn": 1, + "stagnation_events": 0, + "total_completion_tokens": 723, + "total_elapsed_ms": 58860.58, + "total_prompt_tokens": 1576, + "total_retrieved_plane1_tokens": 1181, + "total_tokens": 2299, + "turns": [ + { + "candidate_state_hash": "0a22e8a725d8d1ebb2e9d7dd", + "completion_tokens": 723, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 125.15, + "eval_passed": true, + "failure_signature": "PASS", + "generation_ms": 58786.17, + "inference_oracle_ms": 74.41, + "inference_passed": true, + "passed": true, + "prompt_tokens": 1576, + "response_sha256": "dea191c568e7071eec79de075d80fb2821f6b4adaadacfd0e3188af8fe2e6443", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_19", + "sec5_22", + "sec6_23" + ], + "retrieved_plane1_tokens": 1181, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "family": "B", + "id": "PUB-B-V02", + "inference_certified": true, + "inference_certified_at_turn": 2, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 2, + "pass_at_1": false, + "pass_at_2": true, + "pass_at_3": true, + "pass_at_4": true, + "passed": true, + "refuted_routes": [ + { + "primary_rule": "FLOAT_LITERAL_IN_EXACT_CHECKER", + "state_hash": "31b6cf068d6d", + "turn": 1 + } + ], + "solved_at_turn": 2, + "stagnation_events": 0, + "total_completion_tokens": 1803, + "total_elapsed_ms": 182092.37, + "total_prompt_tokens": 3610, + "total_retrieved_plane1_tokens": 2382, + "total_tokens": 5413, + "turns": [ + { + "candidate_state_hash": "31b6cf068d6de54f7a123357", + "completion_tokens": 993, + "diagnostic_count": 1, + "diagnostic_rules": [ + "FLOAT_LITERAL_IN_EXACT_CHECKER" + ], + "eval_grader_ms": 130.97, + "eval_passed": false, + "failure_signature": "d6625f5b9d6089185ce7a647", + "generation_ms": 89331.06, + "inference_oracle_ms": 77.59, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1512, + "response_sha256": "20a6e44a5d50fe3cf1a559b6546f1394a2d4c37b4632fafaf5477a503451f85d", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_19", + "sec5_22", + "sec6_23" + ], + "retrieved_plane1_tokens": 1181, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "19cf9c661078aaaa16d6caa9", + "completion_tokens": 810, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 126.47, + "eval_passed": true, + "failure_signature": "PASS", + "generation_ms": 92612.86, + "inference_oracle_ms": 70.86, + "inference_passed": true, + "passed": true, + "prompt_tokens": 2098, + "response_sha256": "d3612fa3bd331542916d36f4765f1f7d1a2e92ee4bb6de886b1ab8ac08c29d70", + "retrieved_chunk_ids": [ + "sec12_78", + "sec6_24", + "sec6_25" + ], + "retrieved_plane1_tokens": 1201, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "family": "B", + "id": "PUB-B-V03", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "f6114c203c6a", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "69b9d3c19526", + "turn": 2 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "0219692d0d92", + "turn": 3 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "18b4c6808866", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 3, + "total_completion_tokens": 4096, + "total_elapsed_ms": 291888.73, + "total_prompt_tokens": 6788, + "total_retrieved_plane1_tokens": 4772, + "total_tokens": 10884, + "turns": [ + { + "candidate_state_hash": "f6114c203c6a0dc3cc109a74", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 1.82, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 77900.64, + "inference_oracle_ms": 1.47, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1430, + "response_sha256": "d76a23385fcd86a414f4db6aede7d1124a3589a800fb3f78810d1ad209355bf1", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "69b9d3c195266bab4aff1188", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 1.35, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 73364.1, + "inference_oracle_ms": 1.56, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2088, + "response_sha256": "cd33dee88ae41476fbf60e2a3d69ba8f7f28a7a9d651315281c73070300dad09", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "0219692d0d92be3bc00beaa7", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 8.24, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 67134.35, + "inference_oracle_ms": 2.45, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1627, + "response_sha256": "421cc1674899f76a4332033828f6d8b473a502b71ec2f7a72114660a0e84b324", + "retrieved_chunk_ids": [ + "sec12_69", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1203, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "18b4c6808866417515ea7460", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 1.17, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 73482.62, + "inference_oracle_ms": 1.54, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1643, + "response_sha256": "b0c9163a97d4d4d2727206ad1df3948708618b07466d99f028737bb996e4f208", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_68", + "sec12_72", + "sec6_27" + ], + "retrieved_plane1_tokens": 1205, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "B", + "id": "PUB-B-V04", + "inference_certified": true, + "inference_certified_at_turn": 1, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 1, + "pass_at_1": true, + "pass_at_2": true, + "pass_at_3": true, + "pass_at_4": true, + "passed": true, + "refuted_routes": [], + "solved_at_turn": 1, + "stagnation_events": 0, + "total_completion_tokens": 1007, + "total_elapsed_ms": 78682.92, + "total_prompt_tokens": 1373, + "total_retrieved_plane1_tokens": 1198, + "total_tokens": 2380, + "turns": [ + { + "candidate_state_hash": "7d5f19cf873bd0a7025a60d8", + "completion_tokens": 1007, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 118.36, + "eval_passed": true, + "failure_signature": "PASS", + "generation_ms": 78617.04, + "inference_oracle_ms": 65.88, + "inference_passed": true, + "passed": true, + "prompt_tokens": 1373, + "response_sha256": "e4a15ce809c12fb574c9ef8ce9c606d20e076faed6f71082070ececd1ab5a9f2", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_21", + "sec5_22", + "sec6_25" + ], + "retrieved_plane1_tokens": 1198, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "family": "B", + "id": "PUB-B-V05", + "inference_certified": true, + "inference_certified_at_turn": 1, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 1, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 955, + "total_elapsed_ms": 115798.27, + "total_prompt_tokens": 1412, + "total_retrieved_plane1_tokens": 1173, + "total_tokens": 2367, + "turns": [ + { + "candidate_state_hash": "fee023fe29948d315c429024", + "completion_tokens": 955, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 128.46, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 115709.71, + "inference_oracle_ms": 88.56, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1412, + "response_sha256": "29445da8d2733ab971622e65f822066d5363a7cdc831e8609d01b03d70578566", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_16", + "sec5_18", + "sec5_19", + "sec6_25" + ], + "retrieved_plane1_tokens": 1173, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "family": "B", + "id": "PUB-B-V06", + "inference_certified": true, + "inference_certified_at_turn": 1, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 1, + "pass_at_1": true, + "pass_at_2": true, + "pass_at_3": true, + "pass_at_4": true, + "passed": true, + "refuted_routes": [], + "solved_at_turn": 1, + "stagnation_events": 0, + "total_completion_tokens": 750, + "total_elapsed_ms": 53032.52, + "total_prompt_tokens": 1510, + "total_retrieved_plane1_tokens": 1205, + "total_tokens": 2260, + "turns": [ + { + "candidate_state_hash": "9256edab8d3114008bae8870", + "completion_tokens": 750, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 127.8, + "eval_passed": true, + "failure_signature": "PASS", + "generation_ms": 52963.5, + "inference_oracle_ms": 69.02, + "inference_passed": true, + "passed": true, + "prompt_tokens": 1510, + "response_sha256": "7f3ccfbd427b5365594d0846dda676749e0aeecb385586411d086b8ddbd2d438", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec5_21", + "sec6_25" + ], + "retrieved_plane1_tokens": 1205, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "family": "B", + "id": "PUB-B-V07", + "inference_certified": true, + "inference_certified_at_turn": 2, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 2, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "3ff66f25ca8b", + "turn": 1 + } + ], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 2015, + "total_elapsed_ms": 137879.79, + "total_prompt_tokens": 3231, + "total_retrieved_plane1_tokens": 2364, + "total_tokens": 5246, + "turns": [ + { + "candidate_state_hash": "3ff66f25ca8bb2de9630256e", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 6.05, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 71448.12, + "inference_oracle_ms": 1.1, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1250, + "response_sha256": "739dc5bd657fb383843277aca395af9891e73541ab51ab36f89fb9fe1f1de3e5", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "f44863ed63bec6df5ed37eb5", + "completion_tokens": 991, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 125.96, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 66362.03, + "inference_oracle_ms": 68.54, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1981, + "response_sha256": "e5e6d15089721989bdeaecfc8b7af304c89f044d2774c8b0a2d073932651ff17", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "family": "B", + "id": "PUB-B-V08", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "5bec6142e4bb", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "411064a78ea3", + "turn": 2 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "653ab2575662", + "turn": 3 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "b0a7e811b6dc", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 3, + "total_completion_tokens": 4096, + "total_elapsed_ms": 275452.98, + "total_prompt_tokens": 7409, + "total_retrieved_plane1_tokens": 4832, + "total_tokens": 11505, + "turns": [ + { + "candidate_state_hash": "5bec6142e4bb2a1c6705ceea", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 3.17, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 66858.92, + "inference_oracle_ms": 2.39, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1631, + "response_sha256": "a484dc99f4bae6ae34e649f84aa5975776cb93ded05d773f23fe5f02dab23676", + "retrieved_chunk_ids": [ + "sec5_16", + "sec5_20" + ], + "retrieved_plane1_tokens": 1212, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "411064a78ea3ce4d763f2c34", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 1.75, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 70804.78, + "inference_oracle_ms": 2.0, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2306, + "response_sha256": "6bdbb8ddfa3fc04bf2a50dfcf6275387643bd9b8d75f9e8a10b087dd892ac3e5", + "retrieved_chunk_ids": [ + "sec5_16", + "sec5_20" + ], + "retrieved_plane1_tokens": 1212, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "653ab2575662dbc0081639c6", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 1.76, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 67423.27, + "inference_oracle_ms": 2.14, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1702, + "response_sha256": "1eedeaf6699053f07e3ba6492dbce1f6f85d6453edef8b8e300fb67a71d44f85", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_66", + "sec5_22", + "sec6_26" + ], + "retrieved_plane1_tokens": 1205, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "b0a7e811b6dc68bfb15b2518", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 1.78, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 70356.89, + "inference_oracle_ms": 2.59, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1770, + "response_sha256": "ccdac5f7a8fae02e5a77218b6e981c1a7c8e2641a6f759054892237c1b81c96c", + "retrieved_chunk_ids": [ + "sec12_69", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1203, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "B", + "id": "PUB-B-V09", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "01fb3cf49d32", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "e59f01a2ce4b", + "turn": 2 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "7467b23b8203", + "turn": 3 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "3ed4423571af", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 3, + "total_completion_tokens": 4096, + "total_elapsed_ms": 288278.31, + "total_prompt_tokens": 7880, + "total_retrieved_plane1_tokens": 4816, + "total_tokens": 11976, + "turns": [ + { + "candidate_state_hash": "01fb3cf49d32606ee148f4eb", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 3.41, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 76019.9, + "inference_oracle_ms": 2.64, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1744, + "response_sha256": "e254cbec42c3ea71cd268958ad51bc27ff07b43f4fe1e2f18d88cafc3e61335f", + "retrieved_chunk_ids": [ + "sec5_16", + "sec5_20" + ], + "retrieved_plane1_tokens": 1212, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "e59f01a2ce4b2e668f56e812", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 2.04, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 71073.1, + "inference_oracle_ms": 2.7, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2440, + "response_sha256": "7b18230eff0d6a8aa2a28cc66a90cc715300802efece536b4cea3d057e7ede27", + "retrieved_chunk_ids": [ + "sec5_16", + "sec5_20" + ], + "retrieved_plane1_tokens": 1212, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "7467b23b8203483b12012ae1", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 2.21, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 71989.53, + "inference_oracle_ms": 2.71, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1807, + "response_sha256": "88d3db01177f6630a1cdabc553ff076f1c7f1fd69d46616c175acf28126b4ddf", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec13_84", + "sec6_26" + ], + "retrieved_plane1_tokens": 1185, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "3ed4423571af60542f6e401e", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 2.2, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 69184.04, + "inference_oracle_ms": 3.69, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1889, + "response_sha256": "71b55e3be1deaf5235a7077eb3791cd7f073aada611532af8ff048a5dd85cd59", + "retrieved_chunk_ids": [ + "sec12_74", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1207, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "B", + "id": "PUB-B-V10", + "inference_certified": true, + "inference_certified_at_turn": 3, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 3, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "142f64202eac", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "97a41bf83b23", + "turn": 2 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 2873, + "total_elapsed_ms": 270348.2, + "total_prompt_tokens": 5531, + "total_retrieved_plane1_tokens": 3548, + "total_tokens": 8404, + "turns": [ + { + "candidate_state_hash": "142f64202eac50fead16b446", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 2.41, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 127189.04, + "inference_oracle_ms": 3.43, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1621, + "response_sha256": "3d65151cc5d71fd5964dbe9cc6dba09c450449751aae870b1b44386b1251318d", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_16", + "sec5_18", + "sec5_19", + "sec6_25" + ], + "retrieved_plane1_tokens": 1173, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "97a41bf83b23e4417021cb54", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 1.39, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 79502.99, + "inference_oracle_ms": 2.2, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2190, + "response_sha256": "b5ee8dd6fde7d4682c9c5fa3566177894550ad47056b90315f4310caeb5f06ee", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_16", + "sec5_18", + "sec5_19", + "sec6_25" + ], + "retrieved_plane1_tokens": 1173, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "93d7258d82cb8671f4b4e4d2", + "completion_tokens": 825, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 127.41, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 63578.95, + "inference_oracle_ms": 71.59, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1720, + "response_sha256": "bf9252c95009cc9bc7f8a117460a1bd466d5e8236fa3a0fd959fbb0e99c0344f", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1202, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + } + ], + "turns_used": 3 + }, + { + "family": "B", + "id": "PUB-B-V11", + "inference_certified": true, + "inference_certified_at_turn": 3, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 3, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "a6a18be91150", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "24dbb11a21aa", + "turn": 2 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 2851, + "total_elapsed_ms": 195569.13, + "total_prompt_tokens": 5074, + "total_retrieved_plane1_tokens": 3568, + "total_tokens": 7925, + "turns": [ + { + "candidate_state_hash": "a6a18be9115070b5892db9b8", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 2.37, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 68755.19, + "inference_oracle_ms": 1.36, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1491, + "response_sha256": "97a9a9c34760881f7380f0c4b1cf3a1ac28c284b4e9fe50ba53dea71cff7ca8f", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_16", + "sec5_18", + "sec5_19", + "sec6_25" + ], + "retrieved_plane1_tokens": 1173, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "24dbb11a21aa15b2a8705677", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 1.03, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 72208.46, + "inference_oracle_ms": 1.46, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2087, + "response_sha256": "17c4e6fe9ed2485d92816509c9356d967f51f48806fce0d7f08fb3ab6dd5e146", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_16", + "sec5_18", + "sec5_19", + "sec6_25" + ], + "retrieved_plane1_tokens": 1173, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "9f92eace30ba2878e33e072d", + "completion_tokens": 803, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 120.09, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 54531.39, + "inference_oracle_ms": 71.27, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1496, + "response_sha256": "f6dde906231f66c2aa22dc06a8d0d9a2d5a41ab73f5c0ff8731ae278cfd43515", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1222, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + } + ], + "turns_used": 3 + }, + { + "family": "B", + "id": "PUB-B-V12", + "inference_certified": true, + "inference_certified_at_turn": 1, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 1, + "pass_at_1": true, + "pass_at_2": true, + "pass_at_3": true, + "pass_at_4": true, + "passed": true, + "refuted_routes": [], + "solved_at_turn": 1, + "stagnation_events": 0, + "total_completion_tokens": 763, + "total_elapsed_ms": 57883.44, + "total_prompt_tokens": 1518, + "total_retrieved_plane1_tokens": 1181, + "total_tokens": 2281, + "turns": [ + { + "candidate_state_hash": "0d61869bbb73eb9a8b438c13", + "completion_tokens": 763, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 126.21, + "eval_passed": true, + "failure_signature": "PASS", + "generation_ms": 57813.46, + "inference_oracle_ms": 69.98, + "inference_passed": true, + "passed": true, + "prompt_tokens": 1518, + "response_sha256": "f04aa26d4e6887281dd30361d0e7d1aa6da97f10434e527c4cdd4e6f0a37a378", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_19", + "sec5_22", + "sec6_23" + ], + "retrieved_plane1_tokens": 1181, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "family": "C", + "id": "PUB-C-V01", + "inference_certified": true, + "inference_certified_at_turn": 1, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 1, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 496, + "total_elapsed_ms": 65645.96, + "total_prompt_tokens": 1139, + "total_retrieved_plane1_tokens": 1208, + "total_tokens": 1635, + "turns": [ + { + "candidate_state_hash": "012acfbd601382bdafe5b4e9", + "completion_tokens": 496, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 189.62, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 65645.03, + "inference_oracle_ms": 0.93, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1139, + "response_sha256": "f9f4830cbc8d787dfe3288b5ef8a285186ebe1919013a7037fa9e01da755887a", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "family": "C", + "id": "PUB-C-V02", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PROMPT_EVIDENCE_CONTRADICTION", + "state_hash": "9a6a0ada58c4", + "turn": 1 + }, + { + "primary_rule": "PROMPT_EVIDENCE_CONTRADICTION", + "state_hash": "579762b08828", + "turn": 2 + }, + { + "primary_rule": "PROMPT_EVIDENCE_CONTRADICTION", + "state_hash": "f8343cf93980", + "turn": 3 + }, + { + "primary_rule": "PROMPT_EVIDENCE_CONTRADICTION", + "state_hash": "a586406906be", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 3, + "total_completion_tokens": 3750, + "total_elapsed_ms": 259016.96, + "total_prompt_tokens": 7078, + "total_retrieved_plane1_tokens": 4832, + "total_tokens": 10828, + "turns": [ + { + "candidate_state_hash": "9a6a0ada58c449df8226f133", + "completion_tokens": 944, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION" + ], + "eval_grader_ms": 1.16, + "eval_passed": false, + "failure_signature": "573d12c709756ac67e4eb334", + "generation_ms": 61423.38, + "inference_oracle_ms": 0.47, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1463, + "response_sha256": "8a8c5eee1a79e661c56deb3a543801826e6935452f64b59acfd56025e616dc2d", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "579762b08828c29ab459c6b2", + "completion_tokens": 931, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION" + ], + "eval_grader_ms": 0.61, + "eval_passed": false, + "failure_signature": "573d12c709756ac67e4eb334", + "generation_ms": 67169.8, + "inference_oracle_ms": 0.42, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2264, + "response_sha256": "8af4d5f6feb8163dc552f5dc12be8107792ffa9de16ad4c1d7a7a3fb706143e1", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "f8343cf9398088cb9f283222", + "completion_tokens": 851, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION" + ], + "eval_grader_ms": 0.34, + "eval_passed": false, + "failure_signature": "573d12c709756ac67e4eb334", + "generation_ms": 59010.17, + "inference_oracle_ms": 0.22, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1659, + "response_sha256": "51b83f4488248c82dd4df8c4932c2978df8a17e042351bdf2111fef2a9a89f0b", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "a586406906be8b9158bfc25f", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION" + ], + "eval_grader_ms": 0.79, + "eval_passed": false, + "failure_signature": "573d12c709756ac67e4eb334", + "generation_ms": 71411.81, + "inference_oracle_ms": 0.69, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1692, + "response_sha256": "764b44c7a52522dcfc4a9cdd6d98eaadac9f2b23b804b7adf9e1fabd592b8295", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "C", + "id": "PUB-C-V03", + "inference_certified": true, + "inference_certified_at_turn": 1, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 1, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 707, + "total_elapsed_ms": 49832.23, + "total_prompt_tokens": 1437, + "total_retrieved_plane1_tokens": 1208, + "total_tokens": 2144, + "turns": [ + { + "candidate_state_hash": "ab4576f128e46b1a6e8c1591", + "completion_tokens": 707, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 70.52, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 49832.07, + "inference_oracle_ms": 0.16, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1437, + "response_sha256": "2a3aa8852c76728b66a121de62bc4a243a441fb09ab7407f62fe491fc7812aee", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "family": "C", + "id": "PUB-C-V04", + "inference_certified": true, + "inference_certified_at_turn": 1, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 1, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 580, + "total_elapsed_ms": 44148.35, + "total_prompt_tokens": 1129, + "total_retrieved_plane1_tokens": 1208, + "total_tokens": 1709, + "turns": [ + { + "candidate_state_hash": "c3e9b8351d856c8444d6e83d", + "completion_tokens": 580, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 178.29, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 44148.19, + "inference_oracle_ms": 0.16, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1129, + "response_sha256": "6d85ab7bbc0e79c217072f342daa46728dfc9e93865f8d27e6811cd9bcd0e4d2", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "family": "C", + "id": "PUB-C-V05", + "inference_certified": true, + "inference_certified_at_turn": 2, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 2, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PROMPT_EVIDENCE_CONTRADICTION", + "state_hash": "d85cd07bce09", + "turn": 1 + } + ], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 1685, + "total_elapsed_ms": 119457.16, + "total_prompt_tokens": 3440, + "total_retrieved_plane1_tokens": 2416, + "total_tokens": 5125, + "turns": [ + { + "candidate_state_hash": "d85cd07bce09bbd8f6e34c4a", + "completion_tokens": 745, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PROMPT_EVIDENCE_CONTRADICTION" + ], + "eval_grader_ms": 1.48, + "eval_passed": false, + "failure_signature": "573d12c709756ac67e4eb334", + "generation_ms": 52455.68, + "inference_oracle_ms": 0.17, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1388, + "response_sha256": "8f60be895de294c9f08abef86958a4b6a30fd79923ebb1d2647465bd1680204d", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "d15ef8f12e63f6ca2bc53eff", + "completion_tokens": 940, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 0.5, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 67000.98, + "inference_oracle_ms": 0.33, + "inference_passed": true, + "passed": false, + "prompt_tokens": 2052, + "response_sha256": "56e9e1a1d939387307c1cd025a7f00e4ccde1e45a10e83eab5c2160cf4123caf", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "family": "C", + "id": "PUB-C-V06", + "inference_certified": true, + "inference_certified_at_turn": 4, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "F0_SCHEMA_ERROR", + "state_hash": "a509d3159662", + "turn": 1 + }, + { + "primary_rule": "F0_SCHEMA_ERROR", + "state_hash": "f0ce4adacfc3", + "turn": 2 + }, + { + "primary_rule": "INCOMPLETE_TRUST_BOUNDARY", + "state_hash": "135d32c23df0", + "turn": 3 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 3799, + "total_elapsed_ms": 259493.64, + "total_prompt_tokens": 7593, + "total_retrieved_plane1_tokens": 4806, + "total_tokens": 11392, + "turns": [ + { + "candidate_state_hash": "a509d31596626b8827e20190", + "completion_tokens": 824, + "diagnostic_count": 1, + "diagnostic_rules": [ + "F0_SCHEMA_ERROR" + ], + "eval_grader_ms": 1.38, + "eval_passed": false, + "failure_signature": "e1b53694d7120c7d8b31c4b0", + "generation_ms": 54287.67, + "inference_oracle_ms": 0.18, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1450, + "response_sha256": "25423d3ee130a9a7ec78006f9ba2fc6e0047813faea5b44b955e300311149caa", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "f0ce4adacfc314150bdaf73d", + "completion_tokens": 990, + "diagnostic_count": 1, + "diagnostic_rules": [ + "F0_SCHEMA_ERROR" + ], + "eval_grader_ms": 0.5, + "eval_passed": false, + "failure_signature": "e1b53694d7120c7d8b31c4b0", + "generation_ms": 66638.02, + "inference_oracle_ms": 0.38, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2319, + "response_sha256": "5d483c7f7842aa86eb86eae15d66a7839ec29c41f68391f363a9e6f4b7faee40", + "retrieved_chunk_ids": [ + "sec7_32", + "sec8_36", + "sec8_37", + "sec9_44" + ], + "retrieved_plane1_tokens": 1207, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "135d32c23df0400fc31739b5", + "completion_tokens": 1020, + "diagnostic_count": 1, + "diagnostic_rules": [ + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 76.32, + "eval_passed": false, + "failure_signature": "8fbfbc3d3fca02c956be297e", + "generation_ms": 68323.48, + "inference_oracle_ms": 0.32, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1594, + "response_sha256": "2f55265678f62d5c0abfa30efd81210492895bafc2da676266852a67e86fce65", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_69", + "sec12_72", + "sec12_74", + "sec8_36", + "sec8_37" + ], + "retrieved_plane1_tokens": 1183, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "d1130fec8193d6be5c8e11bb", + "completion_tokens": 965, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 64.39, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 70243.22, + "inference_oracle_ms": 0.37, + "inference_passed": true, + "passed": false, + "prompt_tokens": 2230, + "response_sha256": "21e98a4e65a1b0242116f850567d2f39105cb23a5698d5adbc27742cee2433be", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "C", + "id": "PUB-C-V07", + "inference_certified": true, + "inference_certified_at_turn": 1, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 1, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 542, + "total_elapsed_ms": 35300.39, + "total_prompt_tokens": 1135, + "total_retrieved_plane1_tokens": 1208, + "total_tokens": 1677, + "turns": [ + { + "candidate_state_hash": "84f9ffef50cbf29612faac4b", + "completion_tokens": 542, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 157.95, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 35300.03, + "inference_oracle_ms": 0.36, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1135, + "response_sha256": "21444ff710a58b9fbbfc222c11d97439d074ea4b60e5eacf0108c249ef71a38c", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "family": "C", + "id": "PUB-C-V08", + "inference_certified": true, + "inference_certified_at_turn": 1, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 1, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 917, + "total_elapsed_ms": 61458.93, + "total_prompt_tokens": 1507, + "total_retrieved_plane1_tokens": 1208, + "total_tokens": 2424, + "turns": [ + { + "candidate_state_hash": "4345dd11c970623c0432a116", + "completion_tokens": 917, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 1.51, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 61458.66, + "inference_oracle_ms": 0.27, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1507, + "response_sha256": "8c58773bc9b9a9a66031316e9dd3ea7ffa3580f18bd381a7e30f9abbcc6abc50", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "family": "C", + "id": "PUB-C-V09", + "inference_certified": true, + "inference_certified_at_turn": 1, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 1, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 727, + "total_elapsed_ms": 81783.5, + "total_prompt_tokens": 1449, + "total_retrieved_plane1_tokens": 1208, + "total_tokens": 2176, + "turns": [ + { + "candidate_state_hash": "561fef7bae25ff2deaed834a", + "completion_tokens": 727, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 74.53, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 81783.32, + "inference_oracle_ms": 0.18, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1449, + "response_sha256": "753e7740b7b55f8443540dfed580ac43d71a283520cf4f3905c52cf2ce88cf4d", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "family": "C", + "id": "PUB-C-V10", + "inference_certified": true, + "inference_certified_at_turn": 1, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 1, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 514, + "total_elapsed_ms": 28988.5, + "total_prompt_tokens": 1135, + "total_retrieved_plane1_tokens": 1208, + "total_tokens": 1649, + "turns": [ + { + "candidate_state_hash": "55a1f74e7adb5d2da65edff2", + "completion_tokens": 514, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 163.71, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 28988.12, + "inference_oracle_ms": 0.38, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1135, + "response_sha256": "460b822cc69a8df311cc43f3c8201e63906c23ce55d1386fffefffec04694406", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "family": "C", + "id": "PUB-C-V11", + "inference_certified": true, + "inference_certified_at_turn": 1, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 1, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 821, + "total_elapsed_ms": 55465.71, + "total_prompt_tokens": 1371, + "total_retrieved_plane1_tokens": 1208, + "total_tokens": 2192, + "turns": [ + { + "candidate_state_hash": "9be276087a0c2db4b8a68486", + "completion_tokens": 821, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 1.64, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 55465.44, + "inference_oracle_ms": 0.27, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1371, + "response_sha256": "353eac815a7556462c896afc2c0bf612d615dc0e87dcade4422d98ba5b246194", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "family": "C", + "id": "PUB-C-V12", + "inference_certified": true, + "inference_certified_at_turn": 1, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 1, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 795, + "total_elapsed_ms": 53239.1, + "total_prompt_tokens": 1456, + "total_retrieved_plane1_tokens": 1208, + "total_tokens": 2251, + "turns": [ + { + "candidate_state_hash": "316ade2c93acc5ff8e33945a", + "completion_tokens": 795, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 72.5, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 53238.91, + "inference_oracle_ms": 0.19, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1456, + "response_sha256": "682f5f0df61bcfdd5590088c8830ff218d257ff7cd9c9257d2fdb897a3f724e3", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "family": "D", + "id": "PUB-D-V01", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "19d818a09790", + "turn": 1 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "8df8ffbb55f0", + "turn": 2 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "de571a8cea9a", + "turn": 3 + }, + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "3f96b4c7f3de", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 1598, + "total_elapsed_ms": 110215.19, + "total_prompt_tokens": 7032, + "total_retrieved_plane1_tokens": 4754, + "total_tokens": 8630, + "turns": [ + { + "candidate_state_hash": "19d818a097904735bdc2603d", + "completion_tokens": 453, + "diagnostic_count": 3, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 506.63, + "eval_passed": false, + "failure_signature": "9c71f2d9c5747751ded21e89", + "generation_ms": 28052.6, + "inference_oracle_ms": 459.02, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1319, + "response_sha256": "2ddc51e68f4c4671da9eb2b4da082a40d979bb5445a95fd2f5f7cf63656b12e4", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "8df8ffbb55f021ba9a12bd88", + "completion_tokens": 404, + "diagnostic_count": 1, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 458.74, + "eval_passed": false, + "failure_signature": "c0bd5455dfb6a4bc2799c1a9", + "generation_ms": 28273.13, + "inference_oracle_ms": 443.24, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2057, + "response_sha256": "ab4e1919379c15dc7c40d7fbfe79a30e9172d6efd43930056688bd9bb9af2b74", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "de571a8cea9a5f09fd21b163", + "completion_tokens": 407, + "diagnostic_count": 1, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 485.19, + "eval_passed": false, + "failure_signature": "c0bd5455dfb6a4bc2799c1a9", + "generation_ms": 27753.92, + "inference_oracle_ms": 508.24, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1944, + "response_sha256": "12cee4b7de7a07fb2901ded57793fd8578895cfdd0c719443a3c680031b7668e", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "3f96b4c7f3de6e384fe5889a", + "completion_tokens": 334, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 438.35, + "eval_passed": false, + "failure_signature": "ce0e210a39e79ab778d2808a", + "generation_ms": 24261.02, + "inference_oracle_ms": 464.02, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1712, + "response_sha256": "fff4288fb4989a10d7dd333824742bbde0043a2a0e4e9c713efabb234f80ea38", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_69", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1193, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V02", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "fa5a7d8c440e", + "turn": 1 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "8052e07fc538", + "turn": 2 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "8052e07fc538", + "turn": 3 + }, + { + "primary_rule": "MISSING_CLAIM_TIER_LABEL", + "state_hash": "9427e1ff3647", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 1700, + "total_elapsed_ms": 124262.49, + "total_prompt_tokens": 7044, + "total_retrieved_plane1_tokens": 4754, + "total_tokens": 8744, + "turns": [ + { + "candidate_state_hash": "fa5a7d8c440ec6887dec78a6", + "completion_tokens": 506, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 431.7, + "eval_passed": false, + "failure_signature": "b2b690fce4357f0edb286840", + "generation_ms": 31643.13, + "inference_oracle_ms": 448.48, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1293, + "response_sha256": "1fd1578e5d4c4b8f705171b8b3d8e5376160f6c8deccffea2a4d8c5d0a8d67c3", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "8052e07fc538e35150263769", + "completion_tokens": 444, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "eval_grader_ms": 502.33, + "eval_passed": false, + "failure_signature": "4c3df851138d0cf7779d3e14", + "generation_ms": 33732.5, + "inference_oracle_ms": 600.79, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2049, + "response_sha256": "bfdd50e31d870f68d6a5159a12cc69b10940e706385956b7d8cfc277dc9694f5", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "8052e07fc538e35150263769", + "completion_tokens": 444, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "eval_grader_ms": 470.19, + "eval_passed": false, + "failure_signature": "4c3df851138d0cf7779d3e14", + "generation_ms": 30999.82, + "inference_oracle_ms": 446.52, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1986, + "response_sha256": "bfdd50e31d870f68d6a5159a12cc69b10940e706385956b7d8cfc277dc9694f5", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "9427e1ff3647080200e69491", + "completion_tokens": 306, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_CLAIM_TIER_LABEL" + ], + "eval_grader_ms": 6728.87, + "eval_passed": false, + "failure_signature": "e2cfda7efcae29682df3b19f", + "generation_ms": 19468.31, + "inference_oracle_ms": 6922.94, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1716, + "response_sha256": "d2bae4cd8f372f1a5d5ac29973fd5ad9be93cf9ea86aca8411991d2ea35c1c39", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_69", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1193, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V03", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "0a2eb71d7b60", + "turn": 1 + }, + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "0a2eb71d7b60", + "turn": 2 + }, + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "c2b8f99b0545", + "turn": 3 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "b3faf9a59670", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 2903, + "total_elapsed_ms": 177463.92, + "total_prompt_tokens": 7075, + "total_retrieved_plane1_tokens": 4768, + "total_tokens": 9978, + "turns": [ + { + "candidate_state_hash": "0a2eb71d7b6032fdf11220bd", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 1.39, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 68178.35, + "inference_oracle_ms": 0.03, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1318, + "response_sha256": "465b6024cc2c3182983697c16bc0d2848e2b273a2489ea8cd95f5cf50390a5ae", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "0a2eb71d7b6032fdf11220bd", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 0.57, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 66612.62, + "inference_oracle_ms": 0.07, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2173, + "response_sha256": "465b6024cc2c3182983697c16bc0d2848e2b273a2489ea8cd95f5cf50390a5ae", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "c2b8f99b0545688d46d629c4", + "completion_tokens": 407, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 478.65, + "eval_passed": false, + "failure_signature": "ce0e210a39e79ab778d2808a", + "generation_ms": 27573.03, + "inference_oracle_ms": 493.15, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1607, + "response_sha256": "8d47cd21da138712eea0693ee760bd8da671c7e73dce458895b1cbfcc5381615", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_69", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1192, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "b3faf9a59670367cfce7160b", + "completion_tokens": 448, + "diagnostic_count": 1, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 422.12, + "eval_passed": false, + "failure_signature": "c0bd5455dfb6a4bc2799c1a9", + "generation_ms": 14185.71, + "inference_oracle_ms": 420.96, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1977, + "response_sha256": "9785f658b72ceabbe920e104df7adb33a86d13080792f53c3d76b03467743205", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V04", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "43d659f0c0d3", + "turn": 1 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "447b197e0510", + "turn": 2 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "08dc9d4fcf95", + "turn": 3 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "9578872408ac", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 1836, + "total_elapsed_ms": 124846.45, + "total_prompt_tokens": 7238, + "total_retrieved_plane1_tokens": 4773, + "total_tokens": 9074, + "turns": [ + { + "candidate_state_hash": "43d659f0c0d3e475de7761b7", + "completion_tokens": 474, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 462.73, + "eval_passed": false, + "failure_signature": "b2b690fce4357f0edb286840", + "generation_ms": 28935.06, + "inference_oracle_ms": 453.18, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1334, + "response_sha256": "14ae72f4177ad9259b578d97603236a3d59100a1492b1f204ce2db64def1e0f0", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "447b197e0510170f4e38d139", + "completion_tokens": 446, + "diagnostic_count": 3, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "eval_grader_ms": 435.05, + "eval_passed": false, + "failure_signature": "69e76fbd8ea4b06320e77b79", + "generation_ms": 32116.7, + "inference_oracle_ms": 430.73, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2060, + "response_sha256": "9f3d6af1815eda9d9da7dc1ac8f6eb4061a16f51dde932f6e144b57ff6cc15b9", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "08dc9d4fcf9543d66ef84108", + "completion_tokens": 457, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 450.89, + "eval_passed": false, + "failure_signature": "f8f5cb1a12c896c3823575d7", + "generation_ms": 30151.0, + "inference_oracle_ms": 466.84, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1940, + "response_sha256": "0d5d8262a3c3c1386da594c16a350c36543a8979a6a38c6210ed0bf364df9e3b", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "9578872408ac4afc77142e86", + "completion_tokens": 459, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 459.65, + "eval_passed": false, + "failure_signature": "f8f5cb1a12c896c3823575d7", + "generation_ms": 31824.2, + "inference_oracle_ms": 468.74, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1904, + "response_sha256": "3bcec8a56adb25a93777324bf98cf29149c5a01c68f677125e509e20d98db81f", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec12_66" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V05", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "3884a51dd464", + "turn": 1 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "e5bd76efeb63", + "turn": 2 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "2aed0c2c7b99", + "turn": 3 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "a087bc72185a", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 1746, + "total_elapsed_ms": 138365.76, + "total_prompt_tokens": 6933, + "total_retrieved_plane1_tokens": 4769, + "total_tokens": 8679, + "turns": [ + { + "candidate_state_hash": "3884a51dd4642db8bac35cfc", + "completion_tokens": 489, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 465.36, + "eval_passed": false, + "failure_signature": "b2b690fce4357f0edb286840", + "generation_ms": 35969.3, + "inference_oracle_ms": 453.34, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1302, + "response_sha256": "2c6580b45dc2a674e583ad75818c1c0445ad93544a8de407cfdcd0c4be60473b", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "e5bd76efeb63369534bd9000", + "completion_tokens": 475, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 437.2, + "eval_passed": false, + "failure_signature": "f8f5cb1a12c896c3823575d7", + "generation_ms": 39570.72, + "inference_oracle_ms": 449.14, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2038, + "response_sha256": "4a8bad50e27e8ec8677db27f8b8bc165c3a17f8abfa6fffeff65b740a6eedfc5", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "2aed0c2c7b99f50e53d0a167", + "completion_tokens": 474, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 413.31, + "eval_passed": false, + "failure_signature": "f8f5cb1a12c896c3823575d7", + "generation_ms": 32927.82, + "inference_oracle_ms": 430.37, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1912, + "response_sha256": "11485159a26af8f91800314e7eda76396fc39ac2b7f2684a140da9b71570c54e", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec12_66" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "a087bc72185a69358764145a", + "completion_tokens": 308, + "diagnostic_count": 1, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 446.88, + "eval_passed": false, + "failure_signature": "c0bd5455dfb6a4bc2799c1a9", + "generation_ms": 28142.18, + "inference_oracle_ms": 422.89, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1681, + "response_sha256": "328b2aefb3d4625917b07f30f9b8fb6c2c0f27abae3c345231190e821ef22101", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_69", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1193, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V06", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "81c8fcc37554", + "turn": 1 + }, + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "11d4fedab380", + "turn": 2 + }, + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "7b8f65363a01", + "turn": 3 + }, + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "66666fc07151", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 1566, + "total_elapsed_ms": 110862.16, + "total_prompt_tokens": 7201, + "total_retrieved_plane1_tokens": 4754, + "total_tokens": 8767, + "turns": [ + { + "candidate_state_hash": "81c8fcc37554b6cfcd5bdd0b", + "completion_tokens": 408, + "diagnostic_count": 3, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 443.14, + "eval_passed": false, + "failure_signature": "9c71f2d9c5747751ded21e89", + "generation_ms": 30226.1, + "inference_oracle_ms": 435.22, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1324, + "response_sha256": "c4e68cbc7c7eef2084a0186a6909bb1b2b9730b40de090822fc031393d9acc95", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "11d4fedab3809b03bc1cf1d2", + "completion_tokens": 385, + "diagnostic_count": 3, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "eval_grader_ms": 500.11, + "eval_passed": false, + "failure_signature": "55647d951f2abe4d47d73b96", + "generation_ms": 26118.39, + "inference_oracle_ms": 467.73, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2023, + "response_sha256": "5bcf0176e3f107eb5f35b1060ef7f85d96cb222d0127941d7bf0890c115fae37", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "7b8f65363a01ef0f19314a55", + "completion_tokens": 406, + "diagnostic_count": 3, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "eval_grader_ms": 447.86, + "eval_passed": false, + "failure_signature": "55647d951f2abe4d47d73b96", + "generation_ms": 28401.59, + "inference_oracle_ms": 472.76, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2031, + "response_sha256": "2dfbbea23ee6fe1dd43425846508609e56c6b69f864729a36aafc1f7a96a8964", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "66666fc07151009b7b49a7a4", + "completion_tokens": 367, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 478.88, + "eval_passed": false, + "failure_signature": "ce0e210a39e79ab778d2808a", + "generation_ms": 24251.19, + "inference_oracle_ms": 489.18, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1823, + "response_sha256": "5ab43abd34aa83553e811ffa535ada1a9686193d4e03fb6214ac3b5cbdf7c732", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_69", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1193, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V07", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "1c0fe8946172", + "turn": 1 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "8ceae7d736ea", + "turn": 2 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "48e76bb2c6c8", + "turn": 3 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "8ceae7d736ea", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 2185, + "total_elapsed_ms": 143172.56, + "total_prompt_tokens": 7615, + "total_retrieved_plane1_tokens": 4788, + "total_tokens": 9800, + "turns": [ + { + "candidate_state_hash": "1c0fe8946172062d4c993c32", + "completion_tokens": 520, + "diagnostic_count": 6, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY", + "CLAIM_TIER_INFLATION" + ], + "eval_grader_ms": 422.38, + "eval_passed": false, + "failure_signature": "1c57a54fef892d2734a49b15", + "generation_ms": 34511.12, + "inference_oracle_ms": 517.82, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1313, + "response_sha256": "384853339506fbd79488611a5a79fc561aa110789beb4061002ff6a91ad6fddd", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "8ceae7d736ea3d0d17f8282b", + "completion_tokens": 555, + "diagnostic_count": 4, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 431.92, + "eval_passed": false, + "failure_signature": "3ccdb5e3865db9fb4ff101f1", + "generation_ms": 35495.32, + "inference_oracle_ms": 716.74, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2108, + "response_sha256": "a0b0619d4f8e54dbd8fd90517fdf28c72449cdabf38503556125afb4003890cb", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "48e76bb2c6c8d03e876b334f", + "completion_tokens": 555, + "diagnostic_count": 5, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "CLAIM_TIER_INFLATION" + ], + "eval_grader_ms": 490.05, + "eval_passed": false, + "failure_signature": "469c17db7a6fecc064f9b116", + "generation_ms": 35845.96, + "inference_oracle_ms": 486.08, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2081, + "response_sha256": "d519355e7ae24e6ed71970cb2a934a22a83c909511457f223bb41a182df3f46f", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "8ceae7d736ea3d0d17f8282b", + "completion_tokens": 555, + "diagnostic_count": 4, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 485.38, + "eval_passed": false, + "failure_signature": "3ccdb5e3865db9fb4ff101f1", + "generation_ms": 35105.34, + "inference_oracle_ms": 494.18, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2113, + "response_sha256": "a0b0619d4f8e54dbd8fd90517fdf28c72449cdabf38503556125afb4003890cb", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V08", + "inference_certified": true, + "inference_certified_at_turn": 4, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": true, + "passed": true, + "refuted_routes": [ + { + "primary_rule": "INCOMPLETE_TRUST_BOUNDARY", + "state_hash": "6c4a75b64b29", + "turn": 1 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "4e87dacc723d", + "turn": 2 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "ea3327fffa04", + "turn": 3 + } + ], + "solved_at_turn": 4, + "stagnation_events": 1, + "total_completion_tokens": 1563, + "total_elapsed_ms": 129636.79, + "total_prompt_tokens": 6544, + "total_retrieved_plane1_tokens": 4783, + "total_tokens": 8107, + "turns": [ + { + "candidate_state_hash": "6c4a75b64b2973092956dc79", + "completion_tokens": 391, + "diagnostic_count": 1, + "diagnostic_rules": [ + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 6770.6, + "eval_passed": false, + "failure_signature": "8fbfbc3d3fca02c956be297e", + "generation_ms": 25225.38, + "inference_oracle_ms": 6491.18, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1313, + "response_sha256": "0f779aee535ede7ca80a75bd2f619aee478c0dbe3d9b152c67c0eb2c5b90f1f5", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "4e87dacc723d91bf1f5b9f47", + "completion_tokens": 444, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION" + ], + "eval_grader_ms": 6423.67, + "eval_passed": false, + "failure_signature": "62a77054a7b3e4f4cde66649", + "generation_ms": 27315.61, + "inference_oracle_ms": 6456.56, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1771, + "response_sha256": "aab2a3558b0a37793fff7a2ffb9fbd6d48d0be4ab108d971ee6a99004ee2f6a4", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "ea3327fffa04e8b6ea8bf33e", + "completion_tokens": 430, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION" + ], + "eval_grader_ms": 6633.28, + "eval_passed": false, + "failure_signature": "62a77054a7b3e4f4cde66649", + "generation_ms": 29871.12, + "inference_oracle_ms": 6268.5, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1819, + "response_sha256": "1aa59e4568180f8e8d5e6ffef7b1dd6fc453b1e544de222f25b50e43cfd22693", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec4_09" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "918fc0e82088dedf20785745", + "completion_tokens": 298, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 6550.34, + "eval_passed": true, + "failure_signature": "PASS", + "generation_ms": 21332.89, + "inference_oracle_ms": 6675.55, + "inference_passed": true, + "passed": true, + "prompt_tokens": 1641, + "response_sha256": "615f8765c7f7532c0a4d316dc2601be6b6dc57872178eefb615eff83b0583c9b", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_69", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1192, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V09", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "93bedcb06fba", + "turn": 1 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "a90d77ae73c9", + "turn": 2 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "e4da775868bb", + "turn": 3 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "e4da775868bb", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 2259, + "total_elapsed_ms": 145028.69, + "total_prompt_tokens": 7687, + "total_retrieved_plane1_tokens": 4788, + "total_tokens": 9946, + "turns": [ + { + "candidate_state_hash": "93bedcb06fba8f0662c2b02e", + "completion_tokens": 546, + "diagnostic_count": 6, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY", + "CLAIM_TIER_INFLATION" + ], + "eval_grader_ms": 428.42, + "eval_passed": false, + "failure_signature": "1c57a54fef892d2734a49b15", + "generation_ms": 30400.41, + "inference_oracle_ms": 426.49, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1317, + "response_sha256": "8ef01efef9494dd240cde1074a0fc81286ec3dce0334e185c48ba06ebda9dc02", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "a90d77ae73c989ed10f68872", + "completion_tokens": 571, + "diagnostic_count": 4, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 490.9, + "eval_passed": false, + "failure_signature": "3ccdb5e3865db9fb4ff101f1", + "generation_ms": 38402.86, + "inference_oracle_ms": 461.1, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2140, + "response_sha256": "369b5f682f0e590b991088c48d99dc817a3b745a0dcb781687ae41c35e1c7bf1", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "e4da775868bb4048a07b2955", + "completion_tokens": 571, + "diagnostic_count": 5, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "CLAIM_TIER_INFLATION" + ], + "eval_grader_ms": 488.76, + "eval_passed": false, + "failure_signature": "469c17db7a6fecc064f9b116", + "generation_ms": 40337.46, + "inference_oracle_ms": 454.43, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2099, + "response_sha256": "4e3d7b3989ce01565cc7bbf8e6ef2d345b8137112563e7e1e7b9db1009bc5362", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "e4da775868bb4048a07b2955", + "completion_tokens": 571, + "diagnostic_count": 5, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "CLAIM_TIER_INFLATION" + ], + "eval_grader_ms": 448.98, + "eval_passed": false, + "failure_signature": "469c17db7a6fecc064f9b116", + "generation_ms": 34084.6, + "inference_oracle_ms": 461.34, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2131, + "response_sha256": "4e3d7b3989ce01565cc7bbf8e6ef2d345b8137112563e7e1e7b9db1009bc5362", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V10", + "inference_certified": true, + "inference_certified_at_turn": 4, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": true, + "passed": true, + "refuted_routes": [ + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "99ba854ffa40", + "turn": 1 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "1f62b71d3ea9", + "turn": 2 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "18b28f4eee13", + "turn": 3 + } + ], + "solved_at_turn": 4, + "stagnation_events": 1, + "total_completion_tokens": 1542, + "total_elapsed_ms": 109134.72, + "total_prompt_tokens": 6986, + "total_retrieved_plane1_tokens": 4754, + "total_tokens": 8528, + "turns": [ + { + "candidate_state_hash": "99ba854ffa40d927336f8587", + "completion_tokens": 398, + "diagnostic_count": 1, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 504.43, + "eval_passed": false, + "failure_signature": "c0bd5455dfb6a4bc2799c1a9", + "generation_ms": 23929.04, + "inference_oracle_ms": 490.06, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1330, + "response_sha256": "12badc0d266295f24503daa1c5d2a32a9835a55ffd96796a26c362206ad9626e", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "1f62b71d3ea9687bb30d71af", + "completion_tokens": 403, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "eval_grader_ms": 444.91, + "eval_passed": false, + "failure_signature": "4c3df851138d0cf7779d3e14", + "generation_ms": 27541.4, + "inference_oracle_ms": 437.06, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1932, + "response_sha256": "365883618443d3065806cde3a543913bddabaf11853118c018235e915a4127e4", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "18b28f4eee133b0c586871cb", + "completion_tokens": 404, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "eval_grader_ms": 513.41, + "eval_passed": false, + "failure_signature": "4c3df851138d0cf7779d3e14", + "generation_ms": 29395.46, + "inference_oracle_ms": 491.46, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2003, + "response_sha256": "66fa01e9a3c3de70925ca021b9ecf52a091af80141b8ec1671f0cab88c41be49", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "f80980eae2657e5e7a37328b", + "completion_tokens": 337, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 6305.88, + "eval_passed": true, + "failure_signature": "PASS", + "generation_ms": 20073.66, + "inference_oracle_ms": 6776.58, + "inference_passed": true, + "passed": true, + "prompt_tokens": 1721, + "response_sha256": "e2b7da2e4067010edef9e68b58e677ac0cf2c15c8a6efa6dcbbb468aa193a526", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_69", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1193, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V11", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "305f2ec05e9d", + "turn": 1 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "62f07e55bca6", + "turn": 2 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "3e79407ebf93", + "turn": 3 + }, + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "0f6dfca0a9ca", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 2263, + "total_elapsed_ms": 49174.19, + "total_prompt_tokens": 7356, + "total_retrieved_plane1_tokens": 4769, + "total_tokens": 9619, + "turns": [ + { + "candidate_state_hash": "305f2ec05e9d6a655f91a6de", + "completion_tokens": 465, + "diagnostic_count": 3, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 422.71, + "eval_passed": false, + "failure_signature": "9c71f2d9c5747751ded21e89", + "generation_ms": 9283.47, + "inference_oracle_ms": 404.84, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1319, + "response_sha256": "6e59ebad97030dacf657a5aab425e9729bc5d6041acb97bfe8ed71e53fb7a41d", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "62f07e55bca6d1e3d7739cd1", + "completion_tokens": 646, + "diagnostic_count": 3, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 442.43, + "eval_passed": false, + "failure_signature": "3ccdb5e3865db9fb4ff101f1", + "generation_ms": 13535.23, + "inference_oracle_ms": 438.0, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2080, + "response_sha256": "d1a66a05a5f9faf7aae5d8684e2f5959bda80679db1edd80085d89eba06e1c9b", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "3e79407ebf93a2bd3a15a9c1", + "completion_tokens": 661, + "diagnostic_count": 3, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 464.16, + "eval_passed": false, + "failure_signature": "3ccdb5e3865db9fb4ff101f1", + "generation_ms": 14136.06, + "inference_oracle_ms": 485.34, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2165, + "response_sha256": "15a3d533d9c94429d64a6d504823a9d81ce6c4704efffa3ebb5eb8aa779a538f", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "0f6dfca0a9ca2a1b45462494", + "completion_tokens": 491, + "diagnostic_count": 3, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 402.42, + "eval_passed": false, + "failure_signature": "ce0e210a39e79ab778d2808a", + "generation_ms": 10464.57, + "inference_oracle_ms": 426.68, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1792, + "response_sha256": "72a70c1b48403388717f27733b853e531154ea9481bc562761f93436ed30efa7", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_69", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1193, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V12", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "b8c7dc6bce88", + "turn": 1 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "4e68eb89403a", + "turn": 2 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "4e68eb89403a", + "turn": 3 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "3c8bf6a003b6", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 1849, + "total_elapsed_ms": 128505.73, + "total_prompt_tokens": 7152, + "total_retrieved_plane1_tokens": 4754, + "total_tokens": 9001, + "turns": [ + { + "candidate_state_hash": "b8c7dc6bce8832b4050bfa64", + "completion_tokens": 536, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 422.74, + "eval_passed": false, + "failure_signature": "b2b690fce4357f0edb286840", + "generation_ms": 32993.08, + "inference_oracle_ms": 442.59, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1312, + "response_sha256": "f2d6c36b82601c7a1b266c88740ec4547677c01756ba71009a799a5796913d08", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "4e68eb89403af28f8245870c", + "completion_tokens": 469, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "eval_grader_ms": 419.38, + "eval_passed": false, + "failure_signature": "4c3df851138d0cf7779d3e14", + "generation_ms": 34908.42, + "inference_oracle_ms": 420.98, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2087, + "response_sha256": "17652981a428c45bb436bb5e1a31e16d006f07d2debcbf124c07346f7a459d53", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "4e68eb89403af28f8245870c", + "completion_tokens": 469, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "eval_grader_ms": 442.42, + "eval_passed": false, + "failure_signature": "4c3df851138d0cf7779d3e14", + "generation_ms": 32470.36, + "inference_oracle_ms": 447.72, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2025, + "response_sha256": "17652981a428c45bb436bb5e1a31e16d006f07d2debcbf124c07346f7a459d53", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "3c8bf6a003b65323547727f4", + "completion_tokens": 375, + "diagnostic_count": 1, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 582.81, + "eval_passed": false, + "failure_signature": "c0bd5455dfb6a4bc2799c1a9", + "generation_ms": 26329.53, + "inference_oracle_ms": 493.05, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1728, + "response_sha256": "bef26eb8c183f65c7b98e4ef25c98980e0a799404c327a322b98345fdccb21ad", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_69", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1193, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + } + ], + "label_free_inference_stopping": true, + "max_turns": 4, + "pass_curve": { + "k=1": { + "elapsed_ms_up_to_k": 2640926.5, + "pass_rate_pct": 10.42, + "passed": 5, + "replies_used_up_to_k": 48, + "tokens_used_up_to_k": 98485, + "total": 48 + }, + "k=2": { + "elapsed_ms_up_to_k": 4454668.95, + "pass_rate_pct": 12.5, + "passed": 6, + "replies_used_up_to_k": 81, + "tokens_used_up_to_k": 190318, + "total": 48 + }, + "k=3": { + "elapsed_ms_up_to_k": 6237362.22, + "pass_rate_pct": 12.5, + "passed": 6, + "replies_used_up_to_k": 111, + "tokens_used_up_to_k": 269185, + "total": 48 + }, + "k=4": { + "elapsed_ms_up_to_k": 7869835.55, + "pass_rate_pct": 16.67, + "passed": 8, + "replies_used_up_to_k": 139, + "tokens_used_up_to_k": 340265, + "total": 48 + } + }, + "preservation_probes_8": { + "details": [ + { + "id": "pres_1", + "passed": true, + "reply_preview": "391" + }, + { + "id": "pres_2", + "passed": true, + "reply_preview": "500" + }, + { + "id": "pres_3", + "passed": true, + "reply_preview": "1024" + }, + { + "id": "pres_4", + "passed": true, + "reply_preview": "Tokyo" + }, + { + "id": "pres_5", + "passed": true, + "reply_preview": "No" + }, + { + "id": "pres_6", + "passed": true, + "reply_preview": "len" + }, + { + "id": "pres_7", + "passed": true, + "reply_preview": "404" + }, + { + "id": "pres_8", + "passed": true, + "reply_preview": "Yes" + } + ], + "passed": 8, + "total": 8 + }, + "retrieval_max_tokens": 1250, + "schema": "mcproof-repair-loop-receipt/v0.2", + "split": "validation", + "total_items": 48 +} diff --git a/compile/artifacts/evaluation/repair_loop/validation_repair_p2b.json b/compile/artifacts/evaluation/repair_loop/validation_repair_p2b.json new file mode 100644 index 0000000..70723d6 --- /dev/null +++ b/compile/artifacts/evaluation/repair_loop/validation_repair_p2b.json @@ -0,0 +1,7020 @@ +{ + "arm": "P2-B", + "budget_totals": { + "completion_tokens": 157295, + "inference_certified_items": 14, + "mean_retrieved_plane1_tokens_per_turn": 1195.7, + "oracle_calls": 176, + "prompt_tokens": 309676, + "replies_used": 176, + "stagnation_events": 67, + "total_tokens": 466971 + }, + "by_family": { + "A": { + "k=1": { + "passed": 0, + "total": 12 + }, + "k=2": { + "passed": 0, + "total": 12 + }, + "k=3": { + "passed": 0, + "total": 12 + }, + "k=4": { + "passed": 0, + "total": 12 + } + }, + "B": { + "k=1": { + "passed": 0, + "total": 12 + }, + "k=2": { + "passed": 0, + "total": 12 + }, + "k=3": { + "passed": 0, + "total": 12 + }, + "k=4": { + "passed": 0, + "total": 12 + } + }, + "C": { + "k=1": { + "passed": 0, + "total": 12 + }, + "k=2": { + "passed": 0, + "total": 12 + }, + "k=3": { + "passed": 0, + "total": 12 + }, + "k=4": { + "passed": 0, + "total": 12 + } + }, + "D": { + "k=1": { + "passed": 0, + "total": 12 + }, + "k=2": { + "passed": 0, + "total": 12 + }, + "k=3": { + "passed": 0, + "total": 12 + }, + "k=4": { + "passed": 0, + "total": 12 + } + } + }, + "include_plane1_retrieval": true, + "items": [ + { + "family": "A", + "id": "PUB-A-V01", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "825d75c170c4", + "turn": 1 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "1873393224d8", + "turn": 2 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "10b5fc0c2bed", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "91836ef4bdc8", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 2571, + "total_elapsed_ms": 193976.81, + "total_prompt_tokens": 6369, + "total_retrieved_plane1_tokens": 4805, + "total_tokens": 8940, + "turns": [ + { + "candidate_state_hash": "825d75c170c4f3685662ed8a", + "completion_tokens": 170, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 423.49, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 18526.78, + "inference_oracle_ms": 383.14, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1227, + "response_sha256": "8ad98d0b546e8f378907b99d507492e3b57c532482dbc6d0336388814ebc12f1", + "retrieved_chunk_ids": [ + "sec9_43", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "1873393224d85f67053fa10a", + "completion_tokens": 353, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 365.86, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 23388.8, + "inference_oracle_ms": 338.19, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1478, + "response_sha256": "4b2752f44279239325d17516257d5e6b20cd1f84f974e4dadf611164a91de076", + "retrieved_chunk_ids": [ + "sec9_43", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "10b5fc0c2beddfe8a20e074d", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 348.81, + "eval_passed": false, + "failure_signature": "57846c8d5cc667f02bed28e7", + "generation_ms": 76855.24, + "inference_oracle_ms": 340.42, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1406, + "response_sha256": "9e71785eebeb519e4550ebd007f3c7b08e7ce69c985649f7039be08cb28f5894", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_66", + "sec12_70", + "sec12_78" + ], + "retrieved_plane1_tokens": 1196, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "91836ef4bdc8174f1f9248f9", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 348.53, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 73794.26, + "inference_oracle_ms": 349.98, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2258, + "response_sha256": "e42eba5c8542d43cde2aab52c90c101162bc50215e4da38e82f74b41298f1bfb", + "retrieved_chunk_ids": [ + "sec12_65", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V02", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "41c40578f53a", + "turn": 1 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "61d7d7ab59b0", + "turn": 2 + }, + { + "primary_rule": "NO_THEOREM_DECLARED", + "state_hash": "e3eed8da9193", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "3d71be5cebca", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 4096, + "total_elapsed_ms": 307384.83, + "total_prompt_tokens": 7992, + "total_retrieved_plane1_tokens": 4785, + "total_tokens": 12088, + "turns": [ + { + "candidate_state_hash": "41c40578f53ad178bdc3afa5", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 636.63, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 75027.8, + "inference_oracle_ms": 427.53, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1394, + "response_sha256": "9a7cbbd9bee9265f8add687d18c9cb81cd5929b45e5f1eaa195826f3fe450534", + "retrieved_chunk_ids": [ + "sec4_09", + "sec4_12", + "sec9_43", + "sec9_44" + ], + "retrieved_plane1_tokens": 1194, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "61d7d7ab59b057b73b3f8245", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 422.02, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 82521.17, + "inference_oracle_ms": 383.92, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2449, + "response_sha256": "c0819827c69d43887c655718c9a9c47671a1e6c55c764961f409d365f4a6aa36", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1199, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "e3eed8da9193e8ccca728439", + "completion_tokens": 1024, + "diagnostic_count": 4, + "diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 357.01, + "eval_passed": false, + "failure_signature": "135acee88cd6f947fe803373", + "generation_ms": 73126.79, + "inference_oracle_ms": 375.99, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1771, + "response_sha256": "4fd2750532ad40ad8c0a3041d15a9e7f02244da22cd3b409638597fa4933f260", + "retrieved_chunk_ids": [ + "sec12_67", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1186, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "3d71be5cebcae9da43d4cbaa", + "completion_tokens": 1024, + "diagnostic_count": 3, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 441.3, + "eval_passed": false, + "failure_signature": "7bf11c12f10531dbb6458573", + "generation_ms": 75112.64, + "inference_oracle_ms": 408.99, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2378, + "response_sha256": "01fcb3e0f2585250bdc97511a81605728cdacfe6142a7ce7e896bbcbad62f5cd", + "retrieved_chunk_ids": [ + "sec9_43", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V03", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "a47f192421cf", + "turn": 1 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "19a4f5eca52a", + "turn": 2 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "1cc9a0c9eb08", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "5ea87c52a2a7", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 4096, + "total_elapsed_ms": 304049.02, + "total_prompt_tokens": 6958, + "total_retrieved_plane1_tokens": 4739, + "total_tokens": 11054, + "turns": [ + { + "candidate_state_hash": "a47f192421cf40f3ec276b0d", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 530.03, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 73779.13, + "inference_oracle_ms": 514.33, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1145, + "response_sha256": "ec9acd0d84687efd26d04ac4c8e07de6128aecbb7128b3f857f32a7733243a9d", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec4_09", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1159, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "19a4f5eca52af9c49b156fc4", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 383.2, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 80394.79, + "inference_oracle_ms": 339.22, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2171, + "response_sha256": "d48957d97f519c01d6230ff35f7de8ffd55e2467cc4255ca5654a8f185dc1183", + "retrieved_chunk_ids": [ + "sec12_65", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "1cc9a0c9eb080e5203b03947", + "completion_tokens": 1024, + "diagnostic_count": 4, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 346.22, + "eval_passed": false, + "failure_signature": "a279ed9f1c12c78606eaec87", + "generation_ms": 73131.95, + "inference_oracle_ms": 354.46, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1514, + "response_sha256": "6f161da8dc303422d86f8ffe07857225cf03567d972ee4ededa3ca3dda727332", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_74", + "sec12_76", + "sec12_78" + ], + "retrieved_plane1_tokens": 1186, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "5ea87c52a2a7d98b87462bbc", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 393.17, + "eval_passed": false, + "failure_signature": "7bf11c12f10531dbb6458573", + "generation_ms": 75176.41, + "inference_oracle_ms": 358.73, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2128, + "response_sha256": "58b13a4eb41dd5510af66176c8dcf56a7d10092a68ebfa480076576b56056c1a", + "retrieved_chunk_ids": [ + "sec12_65", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V04", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "a28d9245df3a", + "turn": 1 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "508eb5a48585", + "turn": 2 + }, + { + "primary_rule": "NO_THEOREM_DECLARED", + "state_hash": "02a1746e797b", + "turn": 3 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "2d603945fd89", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 2687, + "total_elapsed_ms": 201542.08, + "total_prompt_tokens": 6353, + "total_retrieved_plane1_tokens": 4816, + "total_tokens": 9040, + "turns": [ + { + "candidate_state_hash": "a28d9245df3a6f73a9f27eff", + "completion_tokens": 288, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 351.37, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 26991.7, + "inference_oracle_ms": 339.98, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1231, + "response_sha256": "880ae7f125f18511359bef9653629e8eb2d9baad12d927016ab9505f66be9800", + "retrieved_chunk_ids": [ + "sec9_43", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "508eb5a48585116360d4503f", + "completion_tokens": 351, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 397.6, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 22790.22, + "inference_oracle_ms": 380.48, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1658, + "response_sha256": "de3ac03b848ae735b0909899ba579092083589ba02176d7bc865e63888210f4c", + "retrieved_chunk_ids": [ + "sec9_43", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "02a1746e797bbdd54e026e9b", + "completion_tokens": 1024, + "diagnostic_count": 3, + "diagnostic_rules": [ + "NO_THEOREM_DECLARED", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 325.12, + "eval_passed": false, + "failure_signature": "135acee88cd6f947fe803373", + "generation_ms": 77691.8, + "inference_oracle_ms": 333.66, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1589, + "response_sha256": "4c63efb6ec08ff0608c6ab6fa5538a71d6244497eff7fabd7804a84d6a22db2f", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_67", + "sec12_69", + "sec12_72", + "sec12_78" + ], + "retrieved_plane1_tokens": 1198, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "2d603945fd896c7519089dfa", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 349.31, + "eval_passed": false, + "failure_signature": "81660a03749a5d7581ebb508", + "generation_ms": 72662.06, + "inference_oracle_ms": 352.18, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1875, + "response_sha256": "6911acb88819d6695aab72f06d545ae0a3b445a03bdcc94aebc0d5903a7b8ea9", + "retrieved_chunk_ids": [ + "sec9_43", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V05", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "NON_INIT_IMPORT", + "state_hash": "9682d15373bf", + "turn": 1 + }, + { + "primary_rule": "FORBIDDEN_META_IMPORT", + "state_hash": "9cb0e2c86a2c", + "turn": 2 + }, + { + "primary_rule": "FORBIDDEN_META_IMPORT", + "state_hash": "04174201e9e4", + "turn": 3 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "2706962817c9", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 3864, + "total_elapsed_ms": 289403.12, + "total_prompt_tokens": 7386, + "total_retrieved_plane1_tokens": 4767, + "total_tokens": 11250, + "turns": [ + { + "candidate_state_hash": "9682d15373bf224cc029a24a", + "completion_tokens": 1024, + "diagnostic_count": 26, + "diagnostic_rules": [ + "NON_INIT_IMPORT", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC" + ], + "eval_grader_ms": 532.45, + "eval_passed": false, + "failure_signature": "22f3b0eae596e84a4837cc07", + "generation_ms": 73850.47, + "inference_oracle_ms": 676.18, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1294, + "response_sha256": "7890ace189afffdc48e3190b4779282e6456b67db28cd9f7c7822f45062d292c", + "retrieved_chunk_ids": [ + "sec12_78", + "sec4_09", + "sec4_10", + "sec9_42", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "9cb0e2c86a2c3f6bc3c4e04c", + "completion_tokens": 961, + "diagnostic_count": 2, + "diagnostic_rules": [ + "FORBIDDEN_META_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 1088.11, + "eval_passed": false, + "failure_signature": "d7d27d1fb93b3c77918fbb87", + "generation_ms": 74463.61, + "inference_oracle_ms": 1136.81, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2284, + "response_sha256": "78ce578e79597c67bef5144d07c592d938fc99da2c7f8b13b848541fb5a04eb1", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1199, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "04174201e9e4e3bc351e939d", + "completion_tokens": 855, + "diagnostic_count": 2, + "diagnostic_rules": [ + "FORBIDDEN_META_IMPORT", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 1171.55, + "eval_passed": false, + "failure_signature": "d7d27d1fb93b3c77918fbb87", + "generation_ms": 64311.44, + "inference_oracle_ms": 1104.06, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2192, + "response_sha256": "bbaa9ca3e6adb784533db8fbd760f21f488dc911e0c54bd62b642d7645827142", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1199, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "2706962817c931aee2d32188", + "completion_tokens": 1024, + "diagnostic_count": 5, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "FORBIDDEN_META_IMPORT", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 352.85, + "eval_passed": false, + "failure_signature": "3389ae2be09eeabcec7097dd", + "generation_ms": 73528.82, + "inference_oracle_ms": 331.73, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1616, + "response_sha256": "d641d592d1acbdfce8228c6051526d231ffaba4624939b7b595a9999ca4c6240", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_71", + "sec9_49" + ], + "retrieved_plane1_tokens": 1187, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V06", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "34e619dbf9da", + "turn": 1 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "f45369dfce61", + "turn": 2 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "06307b21352b", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "1fe180e32bb6", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 3, + "total_completion_tokens": 3825, + "total_elapsed_ms": 280798.36, + "total_prompt_tokens": 5867, + "total_retrieved_plane1_tokens": 4751, + "total_tokens": 9692, + "turns": [ + { + "candidate_state_hash": "34e619dbf9dab7c3c77987e7", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 567.77, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 73851.59, + "inference_oracle_ms": 712.06, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1146, + "response_sha256": "dad37f78ed40af94917d8af3745fee7ba3196cc4df60b2993872538505f6891d", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec4_09", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1174, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "f45369dfce614a83e5ba791e", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 384.45, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 80452.2, + "inference_oracle_ms": 361.48, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1543, + "response_sha256": "88a31a1d943f131ec535986d5e2bc8d007f72a0cd4779fde6c2aa430f537a35d", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1175, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "06307b21352b39be4089142b", + "completion_tokens": 753, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 349.1, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 49859.31, + "inference_oracle_ms": 359.31, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1398, + "response_sha256": "979a546245299dffec02e0c669097a9214f7aa79a4afce6e881adcfd8e8b7c59", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_70", + "sec12_74", + "sec12_78" + ], + "retrieved_plane1_tokens": 1201, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "1fe180e32bb6025aad4fc022", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 337.57, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 74851.93, + "inference_oracle_ms": 350.48, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1780, + "response_sha256": "81c89b9d4360570e2f14c48a676e6aa452260178dec6071e75f0688a1e96bbff", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_67", + "sec12_76", + "sec9_48" + ], + "retrieved_plane1_tokens": 1201, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V07", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "f7e471772df3", + "turn": 1 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "050973d5371d", + "turn": 2 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "e28727bcc1d0", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "568d58e37fb9", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 2, + "total_completion_tokens": 4096, + "total_elapsed_ms": 300576.21, + "total_prompt_tokens": 7376, + "total_retrieved_plane1_tokens": 4702, + "total_tokens": 11472, + "turns": [ + { + "candidate_state_hash": "f7e471772df3a7939638be44", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 505.05, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 73948.37, + "inference_oracle_ms": 654.3, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1222, + "response_sha256": "a74b585ecda4af118f5a0983a1fa827fde4c15b4359e174633efd12b523198f9", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec4_09", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1174, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "050973d5371d4505d9a41cba", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 350.0, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 80581.32, + "inference_oracle_ms": 392.99, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2330, + "response_sha256": "f20698fd5c20c56dec8751620e8d7ad0a85acf9b34b097352fbf0c242a7b5541", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1175, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "e28727bcc1d0098c8e1bc3ac", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 340.94, + "eval_passed": false, + "failure_signature": "7bf11c12f10531dbb6458573", + "generation_ms": 71307.68, + "inference_oracle_ms": 364.19, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1677, + "response_sha256": "d6fe314d69b21f744503ca9569cb5e729bbdee8ccb330e88a05bbc1c85d0ef47", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_67", + "sec12_74", + "sec12_78" + ], + "retrieved_plane1_tokens": 1178, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "568d58e37fb9a12d08e9d702", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 373.67, + "eval_passed": false, + "failure_signature": "7bf11c12f10531dbb6458573", + "generation_ms": 72975.21, + "inference_oracle_ms": 352.15, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2147, + "response_sha256": "07d1905ea447ae4301474fd30458488a2f4166e0fdbd0fcaf908bdc5fad0acfc", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec9_43", + "sec9_44", + "sec9_46", + "sec9_48" + ], + "retrieved_plane1_tokens": 1175, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V08", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "9b824e71e0c8", + "turn": 1 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "31e1967e76b9", + "turn": 2 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "468b783998cc", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "c80ea622a51c", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 3863, + "total_elapsed_ms": 287100.65, + "total_prompt_tokens": 8357, + "total_retrieved_plane1_tokens": 4791, + "total_tokens": 12220, + "turns": [ + { + "candidate_state_hash": "9b824e71e0c81a8ca02d60b3", + "completion_tokens": 1024, + "diagnostic_count": 5, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 471.43, + "eval_passed": false, + "failure_signature": "44879f7925638eff80c061f0", + "generation_ms": 73993.03, + "inference_oracle_ms": 681.16, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1316, + "response_sha256": "828ceb23c189354fcbf772c6597cb6fb256424a0e578a2cbb8b6f1f6779f3a0f", + "retrieved_chunk_ids": [ + "sec4_09", + "sec4_12", + "sec9_43", + "sec9_44" + ], + "retrieved_plane1_tokens": 1194, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "31e1967e76b9f9c71ab0489c", + "completion_tokens": 791, + "diagnostic_count": 5, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 331.23, + "eval_passed": false, + "failure_signature": "09386cd992130552baa135ae", + "generation_ms": 62471.77, + "inference_oracle_ms": 358.6, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2435, + "response_sha256": "46689e442089887091e3154638384cc0a3d4a3f39b632c32b92711b6a9678ae0", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1199, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "468b783998cc644599d5bf30", + "completion_tokens": 1024, + "diagnostic_count": 3, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 343.36, + "eval_passed": false, + "failure_signature": "20da5e4aa99381cc2d01f7f2", + "generation_ms": 74158.33, + "inference_oracle_ms": 357.23, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2304, + "response_sha256": "042646ab1fce2c6356a344e79d7faed946e41b29f663df480ea2855d400c2ba6", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1199, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "c80ea622a51c9820fbd5bf9f", + "completion_tokens": 1024, + "diagnostic_count": 3, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 338.3, + "eval_passed": false, + "failure_signature": "7bf11c12f10531dbb6458573", + "generation_ms": 74744.0, + "inference_oracle_ms": 336.53, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2302, + "response_sha256": "7abdc61c258fada0db4f237cae59ca61df03232e09686767e23d46772ca11ea6", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1199, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V09", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "658bd342d07e", + "turn": 1 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "5d6a7c1ec2ce", + "turn": 2 + }, + { + "primary_rule": "NON_INIT_TACTIC", + "state_hash": "1234544bd9cb", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "2a88e7c477ac", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 4096, + "total_elapsed_ms": 297573.79, + "total_prompt_tokens": 6657, + "total_retrieved_plane1_tokens": 4791, + "total_tokens": 10753, + "turns": [ + { + "candidate_state_hash": "658bd342d07e9932904d59a0", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 385.04, + "eval_passed": false, + "failure_signature": "7bf11c12f10531dbb6458573", + "generation_ms": 75764.33, + "inference_oracle_ms": 386.84, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1312, + "response_sha256": "81cd8f3e738b702afa511f41f1e8cf285c0bb64c1914e183819f6f4d24862737", + "retrieved_chunk_ids": [ + "sec9_43", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "5d6a7c1ec2ce3953905c431c", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 353.79, + "eval_passed": false, + "failure_signature": "7bf11c12f10531dbb6458573", + "generation_ms": 71922.15, + "inference_oracle_ms": 363.23, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1697, + "response_sha256": "573675d9a05147e7fbc071dacaf341714fc232bd8d463d48ddb8d50c43afb0b0", + "retrieved_chunk_ids": [ + "sec9_43", + "sec9_44", + "sec9_48", + "sec9_49" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "1234544bd9cbf675ffe12c7c", + "completion_tokens": 1024, + "diagnostic_count": 5, + "diagnostic_rules": [ + "NON_INIT_TACTIC", + "NON_INIT_TACTIC", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 447.37, + "eval_passed": false, + "failure_signature": "a0f06da330dd763beb7612ce", + "generation_ms": 74543.15, + "inference_oracle_ms": 408.89, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1542, + "response_sha256": "cc1aeddbb907a9ea2496e80f1b28987dd87eccddcec9cdf42185b312e2aac922", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_76", + "sec12_78" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "2a88e7c477acda6a5c0e7d64", + "completion_tokens": 1024, + "diagnostic_count": 3, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 410.61, + "eval_passed": false, + "failure_signature": "7bf11c12f10531dbb6458573", + "generation_ms": 73798.7, + "inference_oracle_ms": 386.5, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2106, + "response_sha256": "65b41409827be5cdbd6d8efddc75808d4179c2ad958634b97fbb6366365f2961", + "retrieved_chunk_ids": [ + "sec12_65", + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_48" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V10", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "88d1df049d59", + "turn": 1 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "2251b734b509", + "turn": 2 + }, + { + "primary_rule": "EXTRA_AXIOM_DECL", + "state_hash": "eb7c26a2ec63", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "8384d42ffe56", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 4096, + "total_elapsed_ms": 294949.15, + "total_prompt_tokens": 6731, + "total_retrieved_plane1_tokens": 4805, + "total_tokens": 10827, + "turns": [ + { + "candidate_state_hash": "88d1df049d5980d740b34349", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 322.18, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 72117.53, + "inference_oracle_ms": 341.73, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1304, + "response_sha256": "a20d2e51a2fab56c153c78e6ee627ab05debe2215a06348a29531d03c0fd73eb", + "retrieved_chunk_ids": [ + "sec4_09", + "sec9_43", + "sec9_53" + ], + "retrieved_plane1_tokens": 1210, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "2251b734b509d391a959027a", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 366.98, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 71046.87, + "inference_oracle_ms": 371.9, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1788, + "response_sha256": "e56abc398afd5232b2a62491644b6f8bff91e38931bd6a5aa06aed217d40ba46", + "retrieved_chunk_ids": [ + "sec12_66", + "sec9_43", + "sec9_53" + ], + "retrieved_plane1_tokens": 1210, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "eb7c26a2ec63d4cc0da00d86", + "completion_tokens": 1024, + "diagnostic_count": 5, + "diagnostic_rules": [ + "EXTRA_AXIOM_DECL", + "UNPROVED_SORRY", + "EXTRA_AXIOM_DECL", + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 386.42, + "eval_passed": false, + "failure_signature": "24b70de0c94fd84983878605", + "generation_ms": 73338.3, + "inference_oracle_ms": 415.22, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1519, + "response_sha256": "7a91f7793515f19ffbdfeaf5821f49569e392ce3f91e1c08d4070474cab92f7c", + "retrieved_chunk_ids": [ + "sec12_69", + "sec12_70", + "sec12_72", + "sec12_74", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1175, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "8384d42ffe5623e491951418", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 418.83, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 76902.14, + "inference_oracle_ms": 415.46, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2120, + "response_sha256": "c8a3b9ac502400da314aa8d6448b16de3aa724e1ae4acd1487318abda54d9fe3", + "retrieved_chunk_ids": [ + "sec12_66", + "sec9_43", + "sec9_53" + ], + "retrieved_plane1_tokens": 1210, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V11", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "20a8a08c9031", + "turn": 1 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "7950935ded40", + "turn": 2 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "9e4d513d8984", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "297fe5b18049", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 4096, + "total_elapsed_ms": 296468.67, + "total_prompt_tokens": 7261, + "total_retrieved_plane1_tokens": 4816, + "total_tokens": 11357, + "turns": [ + { + "candidate_state_hash": "20a8a08c90316ec945ed6d9b", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 313.4, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 72114.97, + "inference_oracle_ms": 341.65, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1304, + "response_sha256": "354ca048feedf2d09296b6094d893c32007e8a65733f00698e5a141648e7e4dc", + "retrieved_chunk_ids": [ + "sec4_09", + "sec9_43", + "sec9_53" + ], + "retrieved_plane1_tokens": 1210, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "7950935ded4037e7b7aab7b0", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 335.01, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 71127.87, + "inference_oracle_ms": 325.99, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2209, + "response_sha256": "9de66d820d0989f27c3b7af26721f3ea9362f933045c19c01777d30f99eb3eb3", + "retrieved_chunk_ids": [ + "sec12_66", + "sec9_43", + "sec9_53" + ], + "retrieved_plane1_tokens": 1210, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "9e4d513d8984fba948e7a5a8", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR", + "MISSING_PROMPT_THEOREM_NAME" + ], + "eval_grader_ms": 381.55, + "eval_passed": false, + "failure_signature": "7bf11c12f10531dbb6458573", + "generation_ms": 75763.32, + "inference_oracle_ms": 474.98, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1663, + "response_sha256": "42b30470c05f3e69bc1bc93a71cd7ce65a003ec83215c489cd85762e4a421a18", + "retrieved_chunk_ids": [ + "sec12_67", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1186, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "297fe5b18049cbd97d87333a", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 398.34, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 75916.85, + "inference_oracle_ms": 403.04, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2085, + "response_sha256": "f9513f8e8e79a1273a03621664ab8b2e45efbe7f397deea060c10e04c271f684", + "retrieved_chunk_ids": [ + "sec12_66", + "sec9_43", + "sec9_53" + ], + "retrieved_plane1_tokens": 1210, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "A", + "id": "PUB-A-V12", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "0a1ccb23e847", + "turn": 1 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "ce9b23aaf05b", + "turn": 2 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "2665b240728d", + "turn": 3 + }, + { + "primary_rule": "LEAN_COMPILE_ERROR", + "state_hash": "a84db80355bf", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 3, + "total_completion_tokens": 2840, + "total_elapsed_ms": 207272.4, + "total_prompt_tokens": 6571, + "total_retrieved_plane1_tokens": 4774, + "total_tokens": 9411, + "turns": [ + { + "candidate_state_hash": "0a1ccb23e847974ef42e653a", + "completion_tokens": 663, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 321.47, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 51955.79, + "inference_oracle_ms": 334.25, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1240, + "response_sha256": "feac215667fd6043730bbe50ff30d8d7a547a523e4bba270def91c473fa0a9a8", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1199, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "ce9b23aaf05b9efc567992e9", + "completion_tokens": 573, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 337.28, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 41006.52, + "inference_oracle_ms": 349.45, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2016, + "response_sha256": "826cb5edb64973412b72cd39e46e85081e34eb5d3135755d4314be424c3d36a9", + "retrieved_chunk_ids": [ + "sec9_42", + "sec9_43", + "sec9_44", + "sec9_46" + ], + "retrieved_plane1_tokens": 1199, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "2665b240728dce0ed0f0b229", + "completion_tokens": 580, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 351.9, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 38337.34, + "inference_oracle_ms": 346.14, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1463, + "response_sha256": "2f3e1067887d999e7bac758f24163e9bb9ed5f9bdaecf4c7425327a1e9e62593", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_70", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1170, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "a84db80355bfb60f7ddcbf7f", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 352.07, + "eval_passed": false, + "failure_signature": "caedb70fc64acb525b99de2c", + "generation_ms": 74584.62, + "inference_oracle_ms": 358.29, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1852, + "response_sha256": "ab68a5a4395b59834fd7931076b4a3785e9e048120bcdff31a6c60fd75ccc780", + "retrieved_chunk_ids": [ + "sec12_67", + "sec12_74", + "sec12_76", + "sec9_48" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "B", + "id": "PUB-B-V01", + "inference_certified": true, + "inference_certified_at_turn": 4, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "61c233ad3300", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "9e72350502b7", + "turn": 2 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "137333001942", + "turn": 3 + } + ], + "solved_at_turn": null, + "stagnation_events": 2, + "total_completion_tokens": 3768, + "total_elapsed_ms": 274369.23, + "total_prompt_tokens": 6966, + "total_retrieved_plane1_tokens": 4781, + "total_tokens": 10734, + "turns": [ + { + "candidate_state_hash": "61c233ad3300062a44850ac5", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 2.28, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 73049.67, + "inference_oracle_ms": 1.41, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1576, + "response_sha256": "f4d7c835c54952306d579d45a908fdab1d6ed725d29fcb5a98755d1f0d6442e0", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_19", + "sec5_22", + "sec6_23" + ], + "retrieved_plane1_tokens": 1181, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "9e72350502b7541b1ae115c5", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 1.25, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 73973.64, + "inference_oracle_ms": 1.87, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2222, + "response_sha256": "9a0805e5793561eaeb4dcf4b18de7e8626a4e8c96a3f22aa5b83a9d4da55df4d", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_16", + "sec5_18", + "sec5_19", + "sec6_25" + ], + "retrieved_plane1_tokens": 1173, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "137333001942bf00a63885eb", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 1.5, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 74629.38, + "inference_oracle_ms": 1.58, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1531, + "response_sha256": "cbf64715d073b4790be31fb89f0bae42ddc9422076b26b65bc006814bb8ccece", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1222, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "2bf1f947f545bdc4ccb4cdd8", + "completion_tokens": 696, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 129.05, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 52635.18, + "inference_oracle_ms": 76.5, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1637, + "response_sha256": "739d1868f6d4219db22e5fe4e46dfdf1a695de721e0493f0a6374cdc1871c48a", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_68", + "sec12_69", + "sec6_27" + ], + "retrieved_plane1_tokens": 1205, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "B", + "id": "PUB-B-V02", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "81d1b72e5362", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "92427ebf3772", + "turn": 2 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "e3093b7bb428", + "turn": 3 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "ffbf7c4baf14", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 2, + "total_completion_tokens": 4096, + "total_elapsed_ms": 301237.99, + "total_prompt_tokens": 7470, + "total_retrieved_plane1_tokens": 4740, + "total_tokens": 11566, + "turns": [ + { + "candidate_state_hash": "81d1b72e5362e128ac8f02b1", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 2.36, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 71461.21, + "inference_oracle_ms": 1.63, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1512, + "response_sha256": "35b448adf4d6571fc89f38760063f68ba11daadf821a8a79b6d3a3f85d9e5cd5", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_19", + "sec5_22", + "sec6_23" + ], + "retrieved_plane1_tokens": 1181, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "92427ebf37729627ad7dc1ee", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 1.18, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 75334.96, + "inference_oracle_ms": 1.78, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2240, + "response_sha256": "358ea56c900946f76304ecf420abcc2dfd997c4db5ab5181a8e185d9bad6bf42", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_19", + "sec5_22", + "sec6_23" + ], + "retrieved_plane1_tokens": 1181, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "e3093b7bb428bc053ef4badc", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 1.4, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 73391.68, + "inference_oracle_ms": 3.52, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1524, + "response_sha256": "6ac3c6ae6d2384ed4c166cebbb46b41806c5acb2baa42811f149e08b44d12531", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_68", + "sec12_69", + "sec6_27" + ], + "retrieved_plane1_tokens": 1205, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "ffbf7c4baf145b768a44f6a9", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 1.86, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 81041.5, + "inference_oracle_ms": 1.71, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2194, + "response_sha256": "07e1fec588581652049641449f5a91893e681807d8bc4ccd52ddb3e360e22e0d", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_16", + "sec5_18", + "sec5_19", + "sec6_25" + ], + "retrieved_plane1_tokens": 1173, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "B", + "id": "PUB-B-V03", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "74f7b5493c17", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "d78d2fab81f4", + "turn": 2 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "411a81ba5173", + "turn": 3 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "6b3a04f60b32", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 4096, + "total_elapsed_ms": 299813.34, + "total_prompt_tokens": 7524, + "total_retrieved_plane1_tokens": 4749, + "total_tokens": 11620, + "turns": [ + { + "candidate_state_hash": "74f7b5493c1725e7caceadf3", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 1.87, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 72175.59, + "inference_oracle_ms": 1.94, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1430, + "response_sha256": "99ab8951b2ef7ba25d46a80a22fe3b72b4c79a69d6e0c0eb891672d6ff6a3c56", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "d78d2fab81f4607639641dba", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 2.52, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 73244.29, + "inference_oracle_ms": 3.25, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2166, + "response_sha256": "53b624c9745b75e7012e55d80f312eeb3bcc34b186dce7d0b1c9a30ea91c108d", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "411a81ba5173d61a3fc055a8", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 3.14, + "eval_passed": false, + "failure_signature": "24143ecebb0328f0d6d1a0f8", + "generation_ms": 75110.62, + "inference_oracle_ms": 3.36, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1625, + "response_sha256": "9df88a8c8db61f749034e685f9580c4291794064a16909a96972de7f2d3964d2", + "retrieved_chunk_ids": [ + "sec12_69", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1203, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "6b3a04f60b324a460bac19a9", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 1.15, + "eval_passed": false, + "failure_signature": "de0a86984f0fc2adf7d8add0", + "generation_ms": 79272.85, + "inference_oracle_ms": 1.44, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2303, + "response_sha256": "2294346901bb3ca12522080683a5f6070be412dd60013442cf9ffe6c47808dad", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "B", + "id": "PUB-B-V04", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "e31ff0762d58", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "c6732f9f62a0", + "turn": 2 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "785902fe90e7", + "turn": 3 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "1453d07c8eef", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 2, + "total_completion_tokens": 4096, + "total_elapsed_ms": 301792.63, + "total_prompt_tokens": 6480, + "total_retrieved_plane1_tokens": 4804, + "total_tokens": 10576, + "turns": [ + { + "candidate_state_hash": "e31ff0762d58486638d624cd", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 0.76, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 72332.64, + "inference_oracle_ms": 0.96, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1373, + "response_sha256": "b741509aeb4f4bbbd0594cf6b663a895ad9f321559fa0b653799426e7d6bc125", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_21", + "sec5_22", + "sec6_25" + ], + "retrieved_plane1_tokens": 1198, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "c6732f9f62a06426a5239d25", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 1.04, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 74547.17, + "inference_oracle_ms": 1.23, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2046, + "response_sha256": "d8e5bcb29bca97da6033ba5cf40b674a80d42ac7cb0d71b6663193c234b897fc", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_21", + "sec5_22", + "sec6_25" + ], + "retrieved_plane1_tokens": 1198, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "785902fe90e77fa62479fe33", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 0.83, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 74431.94, + "inference_oracle_ms": 1.12, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1527, + "response_sha256": "8f760574f45b348d7d71a92060d5adc75480a146ce854c02315437f1422e41bd", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_71", + "sec6_30" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "1453d07c8eefe3fe564dd8f3", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 1.29, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 80474.15, + "inference_oracle_ms": 3.42, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1534, + "response_sha256": "a0debbd990d8d7b3e936f58f92a7154afbb1b9923ca302c232b186ab899dd715", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1202, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "B", + "id": "PUB-B-V05", + "inference_certified": true, + "inference_certified_at_turn": 3, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 3, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "7f3d911b2ae6", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "30c4dfdd58eb", + "turn": 2 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 3016, + "total_elapsed_ms": 230780.88, + "total_prompt_tokens": 4775, + "total_retrieved_plane1_tokens": 3577, + "total_tokens": 7791, + "turns": [ + { + "candidate_state_hash": "7f3d911b2ae645a3edcd0d82", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 2.8, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 79880.64, + "inference_oracle_ms": 1.94, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1412, + "response_sha256": "b9e357dd11163516ee2ba19de4b63d6808fd3fa78d903e3dbbe60199807a4c50", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_16", + "sec5_18", + "sec5_19", + "sec6_25" + ], + "retrieved_plane1_tokens": 1173, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "30c4dfdd58eb79e7acf188e3", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 1.01, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 76801.63, + "inference_oracle_ms": 1.24, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1926, + "response_sha256": "6a2cbf8df7b3f408ec03b0484c7285bf93297c860f8f26998210a650c743ec34", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "6234cd918fb8f2fc8a869e4e", + "completion_tokens": 968, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 122.62, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 74023.41, + "inference_oracle_ms": 72.02, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1437, + "response_sha256": "1c6c64bbf6255f946e76632d055cac2d1137ea0aeb46d5f74a9f8722ebf50d24", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1222, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + } + ], + "turns_used": 3 + }, + { + "family": "B", + "id": "PUB-B-V06", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "0740fabf442d", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "304237676748", + "turn": 2 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "e09a39d2fc1d", + "turn": 3 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "f53d41d0e306", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 2, + "total_completion_tokens": 4096, + "total_elapsed_ms": 306217.74, + "total_prompt_tokens": 7462, + "total_retrieved_plane1_tokens": 4807, + "total_tokens": 11558, + "turns": [ + { + "candidate_state_hash": "0740fabf442ddb025b9bc8d4", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 2.67, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 74467.52, + "inference_oracle_ms": 8.53, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1510, + "response_sha256": "bfacb3cc982b39c2140c40317a13e62dcdbc86288492696f1c07be919822cdb5", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec5_21", + "sec6_25" + ], + "retrieved_plane1_tokens": 1205, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "3042376767487ec5d710ccbb", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 1.86, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 73255.07, + "inference_oracle_ms": 2.57, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2454, + "response_sha256": "fb4b6f04c79da3187dfb22deefea72d4692cc65b9379c445c59c5946aa580b85", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_16", + "sec5_21", + "sec6_25" + ], + "retrieved_plane1_tokens": 1205, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "e09a39d2fc1df531b917ee84", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 1.82, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 77322.05, + "inference_oracle_ms": 1.93, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1722, + "response_sha256": "95d65ccb042ebc3ffca4c57be7bfe5adb00232071aa1b0c017d95ae698ad9e4c", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_71", + "sec12_72", + "sec12_74", + "sec5_22" + ], + "retrieved_plane1_tokens": 1191, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "f53d41d0e3064eedf2e0a290", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 3.17, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 81158.12, + "inference_oracle_ms": 1.95, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1776, + "response_sha256": "06b2ed210e0eb8fc0bb6c419be44b713f35990149e53209e855396d3f7862291", + "retrieved_chunk_ids": [ + "sec12_66", + "sec5_18", + "sec5_19", + "sec6_27" + ], + "retrieved_plane1_tokens": 1206, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "B", + "id": "PUB-B-V07", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "86fb9c83ab7e", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "f883375deca1", + "turn": 2 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "7388c2c16622", + "turn": 3 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "db6558c94be5", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 2, + "total_completion_tokens": 4096, + "total_elapsed_ms": 309586.02, + "total_prompt_tokens": 6497, + "total_retrieved_plane1_tokens": 4734, + "total_tokens": 10593, + "turns": [ + { + "candidate_state_hash": "86fb9c83ab7e9c8ce0b24bf2", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 6.24, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 75160.55, + "inference_oracle_ms": 0.93, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1250, + "response_sha256": "9e9e7b45f4c0f4a9d2970287c795b8fb858b50146b1a98ced4a3fab3d094d389", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "f883375deca1eab2fc4c9f6a", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 4.88, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 72519.48, + "inference_oracle_ms": 0.94, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1976, + "response_sha256": "ddad7206fe1f01074a9dd7bbecbd77f837793e52dcfabadd66d8eb7f324bb57b", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "7388c2c1662265fcf3767753", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 5.63, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 79337.38, + "inference_oracle_ms": 0.97, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1337, + "response_sha256": "b98d8527f01608cbf246de0aa77eaa9c0b9f222c8a7fcaa67f840cdcda3061d7", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_66", + "sec12_69", + "sec12_78", + "sec6_27" + ], + "retrieved_plane1_tokens": 1188, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "db6558c94be55a3d2481c99d", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 5.93, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 82564.92, + "inference_oracle_ms": 0.85, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1934, + "response_sha256": "684b46910a869abb2c906ba7ee50d33fd8ab9a3204fbfe8171c6cb73eca6c02a", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "B", + "id": "PUB-B-V08", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "946ba38e421a", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "a71a00a295d0", + "turn": 2 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "ba13fb9ee94c", + "turn": 3 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "5b4154357c99", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 2, + "total_completion_tokens": 4096, + "total_elapsed_ms": 305824.79, + "total_prompt_tokens": 7435, + "total_retrieved_plane1_tokens": 4832, + "total_tokens": 11531, + "turns": [ + { + "candidate_state_hash": "946ba38e421afef60732ee3e", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 3.31, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 72565.81, + "inference_oracle_ms": 2.81, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1631, + "response_sha256": "6cb3b4cd00f59e2780880d8763afd02234d4026cf4312e42234f68473573bab1", + "retrieved_chunk_ids": [ + "sec5_16", + "sec5_20" + ], + "retrieved_plane1_tokens": 1212, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "a71a00a295d0f1352b57005a", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 2.01, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 78288.31, + "inference_oracle_ms": 2.57, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2330, + "response_sha256": "ee37eedfbb8929e87a34f5206acac3c528c549f53af71e353d9f1656ab476923", + "retrieved_chunk_ids": [ + "sec5_16", + "sec5_20" + ], + "retrieved_plane1_tokens": 1212, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "ba13fb9ee94cb4e042445f51", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 2.72, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 75996.51, + "inference_oracle_ms": 2.91, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1704, + "response_sha256": "aecce07523a5965799690c07ae07744e33ea185a12ec99a1fb3ff4d20566c80e", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_66", + "sec5_22", + "sec6_26" + ], + "retrieved_plane1_tokens": 1205, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "5b4154357c9926c61bbd0eaa", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 1.92, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 78963.56, + "inference_oracle_ms": 2.31, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1770, + "response_sha256": "315a9e6ea02941cdeeb94ada19cd709d7586eb7fd0b5565031aa42af52c58682", + "retrieved_chunk_ids": [ + "sec12_69", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1203, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "B", + "id": "PUB-B-V09", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "85ad1900bd57", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "ebd2c4b6e5da", + "turn": 2 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "489a5f1656da", + "turn": 3 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "18bbd224eff5", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 3, + "total_completion_tokens": 4096, + "total_elapsed_ms": 313107.92, + "total_prompt_tokens": 7969, + "total_retrieved_plane1_tokens": 4816, + "total_tokens": 12065, + "turns": [ + { + "candidate_state_hash": "85ad1900bd5728fae2f2f673", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 4.3, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 76019.53, + "inference_oracle_ms": 5.73, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1744, + "response_sha256": "0c24429bfec29d68a9f8148a5e74dc8374530525b6c4e3b2efd1c0f64eb2daa4", + "retrieved_chunk_ids": [ + "sec5_16", + "sec5_20" + ], + "retrieved_plane1_tokens": 1212, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "ebd2c4b6e5da92b7a68fda4c", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 2.28, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 78219.41, + "inference_oracle_ms": 3.27, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2530, + "response_sha256": "f5eed912a11694048001a54f557d902fae5a3e6d157b5cdf71fd1d8fbbcecac2", + "retrieved_chunk_ids": [ + "sec5_16", + "sec5_20" + ], + "retrieved_plane1_tokens": 1212, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "489a5f1656da1d4ade85183a", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 2.37, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 77973.81, + "inference_oracle_ms": 2.86, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1807, + "response_sha256": "fc15bc54e6b9cf444985e7be025ab020394611f2585889314875121c85cf3fd2", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec13_84", + "sec6_26" + ], + "retrieved_plane1_tokens": 1185, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "18bbd224eff530fa7b45ccdb", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 2.84, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 80878.25, + "inference_oracle_ms": 5.06, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1888, + "response_sha256": "2a73fd097ac5b0018e42e970294bdc1b2a5842863080b408c67b5e963f4c06dc", + "retrieved_chunk_ids": [ + "sec12_74", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1207, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "B", + "id": "PUB-B-V10", + "inference_certified": true, + "inference_certified_at_turn": 3, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 3, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "266faf23fd22", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "ff790cae92ed", + "turn": 2 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 2623, + "total_elapsed_ms": 195166.96, + "total_prompt_tokens": 5625, + "total_retrieved_plane1_tokens": 3587, + "total_tokens": 8248, + "turns": [ + { + "candidate_state_hash": "266faf23fd2237ef712184ce", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 2.7, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 75631.97, + "inference_oracle_ms": 1.84, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1621, + "response_sha256": "13623fcebe8ed310535cde73b36c88d24e87216cdd37a79db9906a4cb9f5943b", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_16", + "sec5_18", + "sec5_19", + "sec6_25" + ], + "retrieved_plane1_tokens": 1173, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "ff790cae92edd68a26361706", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 2.81, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 76893.55, + "inference_oracle_ms": 3.92, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2269, + "response_sha256": "afadfc3c7dfd9c08e350acac8c55b55366bbc777a4df928ae1b2f67438bf2169", + "retrieved_chunk_ids": [ + "sec5_16", + "sec5_20" + ], + "retrieved_plane1_tokens": 1212, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "ea71ca7dcb1ca4d407894c33", + "completion_tokens": 575, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 116.59, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 42569.99, + "inference_oracle_ms": 65.69, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1735, + "response_sha256": "49b47e6b2dbd9cb739f519bbbe339b06f1273261d783c406cb15df6709dce312", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec6_24" + ], + "retrieved_plane1_tokens": 1202, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + } + ], + "turns_used": 3 + }, + { + "family": "B", + "id": "PUB-B-V11", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "66e06d1b366e", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "56771645941c", + "turn": 2 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "aba0c9dc6335", + "turn": 3 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "a55f0e27990b", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 2, + "total_completion_tokens": 4096, + "total_elapsed_ms": 314109.38, + "total_prompt_tokens": 6737, + "total_retrieved_plane1_tokens": 4773, + "total_tokens": 10833, + "turns": [ + { + "candidate_state_hash": "66e06d1b366eaed7da95b829", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 2.4, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 76236.58, + "inference_oracle_ms": 1.47, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1491, + "response_sha256": "1aaab99cc76dd9c4d2b1633b21b8e5fd986ad2971efceca1e71ad408cfc073c0", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_16", + "sec5_18", + "sec5_19", + "sec6_25" + ], + "retrieved_plane1_tokens": 1173, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "56771645941cbaf47525fb81", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 1.23, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 75592.67, + "inference_oracle_ms": 1.42, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2103, + "response_sha256": "ba4a7dedc84c8d5a45b48d28554564340e0420213c9288303dfb706950454bec", + "retrieved_chunk_ids": [ + "sec11_61", + "sec5_15", + "sec5_16", + "sec5_18", + "sec5_19", + "sec6_25" + ], + "retrieved_plane1_tokens": 1173, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "aba0c9dc63359e9b5fb6d899", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 1.19, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 79103.76, + "inference_oracle_ms": 1.61, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1520, + "response_sha256": "111d2fb4d5226a83215d27ecee0de4d17a6be11843fbfd422d5c3aa00595a894", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1222, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "a55f0e27990be6fe0e3c8235", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR" + ], + "eval_grader_ms": 1.51, + "eval_passed": false, + "failure_signature": "20028b0209c91e67db556a7e", + "generation_ms": 83169.3, + "inference_oracle_ms": 2.57, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1623, + "response_sha256": "18a1843ba91c558b80eda7aa9a2ca5976282a35e833f13703f4d808b74cd6723", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_68", + "sec12_69", + "sec6_27" + ], + "retrieved_plane1_tokens": 1205, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "B", + "id": "PUB-B-V12", + "inference_certified": true, + "inference_certified_at_turn": 4, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "d4ba24653389", + "turn": 1 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "f49967065ede", + "turn": 2 + }, + { + "primary_rule": "PYTHON_SYNTAX_ERROR", + "state_hash": "aedc59f97c07", + "turn": 3 + } + ], + "solved_at_turn": null, + "stagnation_events": 2, + "total_completion_tokens": 3951, + "total_elapsed_ms": 300344.27, + "total_prompt_tokens": 6563, + "total_retrieved_plane1_tokens": 4769, + "total_tokens": 10514, + "turns": [ + { + "candidate_state_hash": "d4ba24653389163c5461cf41", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 1.53, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 76229.41, + "inference_oracle_ms": 1.36, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1518, + "response_sha256": "bb5edd7af89e902b152f8c196439ceeb5613aa7396f831f0bd9ac0bb251cbe7f", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_19", + "sec5_22", + "sec6_23" + ], + "retrieved_plane1_tokens": 1181, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "f49967065ede0f48b37e998b", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 1.52, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 76176.07, + "inference_oracle_ms": 1.41, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1997, + "response_sha256": "5497c7b07626f3e382515d28994d09432a2fc0ad1c1c4fca6c12e3d32cdc7ecd", + "retrieved_chunk_ids": [ + "sec5_15", + "sec5_18", + "sec5_20", + "sec6_25" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "aedc59f97c07e391fa0780ef", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PYTHON_SYNTAX_ERROR", + "MISSING_ENTRY_FUNCTION" + ], + "eval_grader_ms": 0.98, + "eval_passed": false, + "failure_signature": "b74a0bb0ae9ce9e885e01c96", + "generation_ms": 76071.07, + "inference_oracle_ms": 1.27, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1469, + "response_sha256": "e4d806979b06d4b63f7be3afbfff2e0651934134b0ff006c5a4ab3dc400a0f93", + "retrieved_chunk_ids": [ + "sec12_78", + "sec5_17" + ], + "retrieved_plane1_tokens": 1222, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "93d52d5ced61ddb386d81fd4", + "completion_tokens": 879, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 243.53, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 71745.2, + "inference_oracle_ms": 118.48, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1579, + "response_sha256": "0a64a1aea53cc5aa3e09228522f75c6f1b57ab28239c0fc02fdf34abb5ce4495", + "retrieved_chunk_ids": [ + "sec12_65", + "sec12_66", + "sec12_69", + "sec5_21", + "sec6_30" + ], + "retrieved_plane1_tokens": 1184, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "C", + "id": "PUB-C-V01", + "inference_certified": true, + "inference_certified_at_turn": 3, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 3, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "MISSING_WHAT_IS_NOT_PROVED", + "state_hash": "f6e5075095ea", + "turn": 1 + }, + { + "primary_rule": "MISSING_WHAT_IS_NOT_PROVED", + "state_hash": "a951ef9e6b80", + "turn": 2 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 2555, + "total_elapsed_ms": 201557.99, + "total_prompt_tokens": 4290, + "total_retrieved_plane1_tokens": 3597, + "total_tokens": 6845, + "turns": [ + { + "candidate_state_hash": "f6e5075095eaeddca236c572", + "completion_tokens": 858, + "diagnostic_count": 2, + "diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "eval_grader_ms": 185.85, + "eval_passed": false, + "failure_signature": "3668bc04ca209bbe827dbfdd", + "generation_ms": 61952.46, + "inference_oracle_ms": 0.26, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1139, + "response_sha256": "cb548a2dc09aab4dfddc0c35c21a8e9b0193fb7cfa0402c241af25454e118702", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "a951ef9e6b80f4ad3f3c3d0e", + "completion_tokens": 673, + "diagnostic_count": 2, + "diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "eval_grader_ms": 189.93, + "eval_passed": false, + "failure_signature": "3668bc04ca209bbe827dbfdd", + "generation_ms": 58090.41, + "inference_oracle_ms": 0.22, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1776, + "response_sha256": "ad84db5f76ed9bca905a9d7a54a40f2a2b6d7a7e402a256345edb4db8b1b68e1", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "b08b55c9333a7f682f7159d5", + "completion_tokens": 1024, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 181.28, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 81514.46, + "inference_oracle_ms": 0.18, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1375, + "response_sha256": "be6c150b79f320eb324479122b084313d8a79daee6966587adc9a9c405ae10a9", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec7_32", + "sec7_34" + ], + "retrieved_plane1_tokens": 1181, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + } + ], + "turns_used": 3 + }, + { + "family": "C", + "id": "PUB-C-V02", + "inference_certified": true, + "inference_certified_at_turn": 2, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 2, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "INVALID_JSON_BLOCK", + "state_hash": "a3de6087992c", + "turn": 1 + } + ], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 2048, + "total_elapsed_ms": 170009.53, + "total_prompt_tokens": 3684, + "total_retrieved_plane1_tokens": 2416, + "total_tokens": 5732, + "turns": [ + { + "candidate_state_hash": "a3de6087992c2e4013ac6828", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "INVALID_JSON_BLOCK" + ], + "eval_grader_ms": 0.79, + "eval_passed": false, + "failure_signature": "8b5f85706efaa7b5838dea27", + "generation_ms": 87392.73, + "inference_oracle_ms": 0.36, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1463, + "response_sha256": "6eb2bf9e3d81f4dbf60dca99bb00b3b7087e8fbed2ae3ab59ccdca880de52ea9", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "313efae4fd468c4d0fcf5221", + "completion_tokens": 1024, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 0.59, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 82615.92, + "inference_oracle_ms": 0.52, + "inference_passed": true, + "passed": false, + "prompt_tokens": 2221, + "response_sha256": "1e54a826db960b9ce8b99a1d45dfedcde947918a68b73c178ef55649821870c6", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "family": "C", + "id": "PUB-C-V03", + "inference_certified": true, + "inference_certified_at_turn": 2, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 2, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "INCOMPLETE_TRUST_BOUNDARY", + "state_hash": "f52ea51d15ef", + "turn": 1 + } + ], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 1800, + "total_elapsed_ms": 145972.17, + "total_prompt_tokens": 3675, + "total_retrieved_plane1_tokens": 2416, + "total_tokens": 5475, + "turns": [ + { + "candidate_state_hash": "f52ea51d15ef785dadb4fee4", + "completion_tokens": 793, + "diagnostic_count": 1, + "diagnostic_rules": [ + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 85.11, + "eval_passed": false, + "failure_signature": "8fbfbc3d3fca02c956be297e", + "generation_ms": 66502.11, + "inference_oracle_ms": 0.19, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1437, + "response_sha256": "cc0cd6b56cb601224841c9a84cb2ff4d0da35f7a258124fd4f913dd259e1b6ef", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "9714c4c844de79282388b227", + "completion_tokens": 1007, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 68.49, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 79469.69, + "inference_oracle_ms": 0.18, + "inference_passed": true, + "passed": false, + "prompt_tokens": 2238, + "response_sha256": "fa9e36891f7ddbfe1342be95438977066f4e84b01cab194e6944199ca06e0718", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "family": "C", + "id": "PUB-C-V04", + "inference_certified": true, + "inference_certified_at_turn": 4, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "MISSING_WHAT_IS_NOT_PROVED", + "state_hash": "03c0e696708a", + "turn": 1 + }, + { + "primary_rule": "MISSING_WHAT_IS_NOT_PROVED", + "state_hash": "bcb11ae1baae", + "turn": 2 + }, + { + "primary_rule": "MISSING_WHAT_IS_NOT_PROVED", + "state_hash": "b9a62c2fba2a", + "turn": 3 + } + ], + "solved_at_turn": null, + "stagnation_events": 2, + "total_completion_tokens": 2994, + "total_elapsed_ms": 233089.67, + "total_prompt_tokens": 5739, + "total_retrieved_plane1_tokens": 4788, + "total_tokens": 8733, + "turns": [ + { + "candidate_state_hash": "03c0e696708a2e15a15014c5", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "eval_grader_ms": 151.07, + "eval_passed": false, + "failure_signature": "3668bc04ca209bbe827dbfdd", + "generation_ms": 77051.91, + "inference_oracle_ms": 0.29, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1129, + "response_sha256": "fb098039a76cda4c640234e9da744714cf2245cd8f69b3c8df6220ec99253130", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "bcb11ae1baae200e513129a0", + "completion_tokens": 477, + "diagnostic_count": 2, + "diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "eval_grader_ms": 217.1, + "eval_passed": false, + "failure_signature": "3668bc04ca209bbe827dbfdd", + "generation_ms": 35997.74, + "inference_oracle_ms": 0.23, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1837, + "response_sha256": "1610a53e11e9d86305231fd50112d4d2c0412819d16c53b6e7d6db044e41b15c", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "b9a62c2fba2a07e7f8961b87", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "eval_grader_ms": 205.86, + "eval_passed": false, + "failure_signature": "3668bc04ca209bbe827dbfdd", + "generation_ms": 81082.77, + "inference_oracle_ms": 0.36, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1367, + "response_sha256": "09e790d9c1a2e82e899a6147eb656f5548a551537715d205feaf96da0c7db541", + "retrieved_chunk_ids": [ + "sec12_69", + "sec12_78", + "sec7_32", + "sec7_34" + ], + "retrieved_plane1_tokens": 1183, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "0fcbacbaa068bfec90da3d6c", + "completion_tokens": 469, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 170.58, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 38956.2, + "inference_oracle_ms": 0.17, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1406, + "response_sha256": "7ec67e9809cbc07fd2e6f3b9e2e0a15203a8cb9f9071ae452cc6eb5f925944ed", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec12_78", + "sec7_32", + "sec7_35" + ], + "retrieved_plane1_tokens": 1189, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "C", + "id": "PUB-C-V05", + "inference_certified": true, + "inference_certified_at_turn": 2, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 2, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "MISSING_WHAT_IS_NOT_PROVED", + "state_hash": "ab762debc3a4", + "turn": 1 + } + ], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 1960, + "total_elapsed_ms": 162056.72, + "total_prompt_tokens": 3484, + "total_retrieved_plane1_tokens": 2416, + "total_tokens": 5444, + "turns": [ + { + "candidate_state_hash": "ab762debc3a452f6d3ac24dd", + "completion_tokens": 1024, + "diagnostic_count": 3, + "diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST", + "MISSING_REQUIRED_JSON_BLOCK" + ], + "eval_grader_ms": 1.35, + "eval_passed": false, + "failure_signature": "79ae30e4ebfd52a97771e0df", + "generation_ms": 79899.7, + "inference_oracle_ms": 0.32, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1388, + "response_sha256": "f8668a943d8eeab01d0340dee27dcd1715c1dd0fad3dca973d558ef4e355d194", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "6ff4f7a1671d32bee0c7b9b2", + "completion_tokens": 936, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 0.53, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 82156.32, + "inference_oracle_ms": 0.38, + "inference_passed": true, + "passed": false, + "prompt_tokens": 2096, + "response_sha256": "88592bb32f0c93ad5de6832abaf358873050a3dabd088ec5e24462c462c31c08", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "family": "C", + "id": "PUB-C-V06", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "INVALID_JSON_BLOCK", + "state_hash": "f3289bfbd5ed", + "turn": 1 + }, + { + "primary_rule": "INVALID_JSON_BLOCK", + "state_hash": "9d83e14656c0", + "turn": 2 + }, + { + "primary_rule": "INCOMPLETE_TRUST_BOUNDARY", + "state_hash": "be08a1b0a5da", + "turn": 3 + }, + { + "primary_rule": "INVALID_JSON_BLOCK", + "state_hash": "061495f6578c", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 2963, + "total_elapsed_ms": 242776.18, + "total_prompt_tokens": 7635, + "total_retrieved_plane1_tokens": 4816, + "total_tokens": 10598, + "turns": [ + { + "candidate_state_hash": "f3289bfbd5ed973427c524d4", + "completion_tokens": 824, + "diagnostic_count": 1, + "diagnostic_rules": [ + "INVALID_JSON_BLOCK" + ], + "eval_grader_ms": 2.69, + "eval_passed": false, + "failure_signature": "8b5f85706efaa7b5838dea27", + "generation_ms": 70579.48, + "inference_oracle_ms": 0.21, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1450, + "response_sha256": "8a22831c4e79fc82b84203b4a47d9e2d07938b26842cdb56a741316e7b9a674c", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "9d83e14656c0002154857c9c", + "completion_tokens": 851, + "diagnostic_count": 1, + "diagnostic_rules": [ + "INVALID_JSON_BLOCK" + ], + "eval_grader_ms": 1.38, + "eval_passed": false, + "failure_signature": "8b5f85706efaa7b5838dea27", + "generation_ms": 62336.65, + "inference_oracle_ms": 0.25, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2271, + "response_sha256": "f3a3ff1a5b0a31f68b502ba6ca1a2e5255dbabddc1fdf671aa3167b92b1bbbc4", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "be08a1b0a5dab60b1a4d000c", + "completion_tokens": 686, + "diagnostic_count": 2, + "diagnostic_rules": [ + "INCOMPLETE_TRUST_BOUNDARY", + "INVALID_JSON_BLOCK" + ], + "eval_grader_ms": 0.44, + "eval_passed": false, + "failure_signature": "cd8394c3d9299a43e870300a", + "generation_ms": 57407.4, + "inference_oracle_ms": 0.26, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1685, + "response_sha256": "bc39691a93ffb38fbc64d120b0b74bddaaa63c9db8a45c6cb4978b55bc203949", + "retrieved_chunk_ids": [ + "sec12_69", + "sec12_72", + "sec12_78", + "sec8_36", + "sec8_38" + ], + "retrieved_plane1_tokens": 1192, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "061495f6578c1508230e0f11", + "completion_tokens": 602, + "diagnostic_count": 1, + "diagnostic_rules": [ + "INVALID_JSON_BLOCK" + ], + "eval_grader_ms": 0.33, + "eval_passed": false, + "failure_signature": "8b5f85706efaa7b5838dea27", + "generation_ms": 52451.77, + "inference_oracle_ms": 0.16, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2229, + "response_sha256": "910cf74786d4f40fa21ac6c992746b4bd795ab7a9547128025fc1ddfae7086d8", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "C", + "id": "PUB-C-V07", + "inference_certified": true, + "inference_certified_at_turn": 3, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 3, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "MISSING_WHAT_IS_NOT_PROVED", + "state_hash": "cd5793cafc18", + "turn": 1 + }, + { + "primary_rule": "MISSING_WHAT_IS_NOT_PROVED", + "state_hash": "85f077046e61", + "turn": 2 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 2038, + "total_elapsed_ms": 169179.27, + "total_prompt_tokens": 4244, + "total_retrieved_plane1_tokens": 3597, + "total_tokens": 6282, + "turns": [ + { + "candidate_state_hash": "cd5793cafc18e2a42116d1ee", + "completion_tokens": 514, + "diagnostic_count": 2, + "diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "eval_grader_ms": 193.08, + "eval_passed": false, + "failure_signature": "3668bc04ca209bbe827dbfdd", + "generation_ms": 39874.6, + "inference_oracle_ms": 0.31, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1135, + "response_sha256": "e476b34b962a308dfc0368d79d63af01a4aec6b81f2ce420614b3da60c8f707c", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "85f077046e61370af9ed5f11", + "completion_tokens": 500, + "diagnostic_count": 2, + "diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "eval_grader_ms": 155.69, + "eval_passed": false, + "failure_signature": "3668bc04ca209bbe827dbfdd", + "generation_ms": 46702.08, + "inference_oracle_ms": 0.24, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1737, + "response_sha256": "5d7ae9fca155e9ea203f843d5c741a1af97578408b22ff7a1c9057317775ef5e", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "1d1f45e5604f4cd5e65fee42", + "completion_tokens": 1024, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 160.95, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 82601.79, + "inference_oracle_ms": 0.25, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1372, + "response_sha256": "7d2a3863f20be08082b04c1cd16ce9423bcb36272567ce8c276b4139c5b3faa1", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_78", + "sec7_32", + "sec7_34" + ], + "retrieved_plane1_tokens": 1181, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + } + ], + "turns_used": 3 + }, + { + "family": "C", + "id": "PUB-C-V08", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "INVALID_JSON_BLOCK", + "state_hash": "251983cd088e", + "turn": 1 + }, + { + "primary_rule": "INVALID_JSON_BLOCK", + "state_hash": "56b8cedff8f3", + "turn": 2 + }, + { + "primary_rule": "MISSING_WHAT_YOU_MUST_TRUST", + "state_hash": "04c35693a0a7", + "turn": 3 + }, + { + "primary_rule": "MISSING_WHAT_YOU_MUST_TRUST", + "state_hash": "eb33a2056441", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 2, + "total_completion_tokens": 3940, + "total_elapsed_ms": 330263.54, + "total_prompt_tokens": 7756, + "total_retrieved_plane1_tokens": 4824, + "total_tokens": 11696, + "turns": [ + { + "candidate_state_hash": "251983cd088e7221ba563b62", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "INVALID_JSON_BLOCK" + ], + "eval_grader_ms": 1.4, + "eval_passed": false, + "failure_signature": "8b5f85706efaa7b5838dea27", + "generation_ms": 85525.98, + "inference_oracle_ms": 0.23, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1507, + "response_sha256": "d2c3259dfde8c30525f8c6a0ab6d81876ac16d2b22f43ea5c19d9f4ae92f4a03", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "56b8cedff8f36b5e9b08276e", + "completion_tokens": 868, + "diagnostic_count": 1, + "diagnostic_rules": [ + "INVALID_JSON_BLOCK" + ], + "eval_grader_ms": 0.51, + "eval_passed": false, + "failure_signature": "8b5f85706efaa7b5838dea27", + "generation_ms": 71682.21, + "inference_oracle_ms": 0.31, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2187, + "response_sha256": "cb09755807f2bd68d92b299dcca07afad3c3b53a179209c423adfbab08140484", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "04c35693a0a7bfbccdcfb50a", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "eval_grader_ms": 0.7, + "eval_passed": false, + "failure_signature": "6c8ed7bc05da388de244a1b8", + "generation_ms": 82882.96, + "inference_oracle_ms": 0.5, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1749, + "response_sha256": "9c244574ff84f49f691cadbe323cde92a35fb92bb5797762547b4b85abf91054", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_72", + "sec12_78", + "sec8_39" + ], + "retrieved_plane1_tokens": 1200, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "eb33a205644146d77b367cb8", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "eval_grader_ms": 1.74, + "eval_passed": false, + "failure_signature": "6c8ed7bc05da388de244a1b8", + "generation_ms": 90170.84, + "inference_oracle_ms": 0.51, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2313, + "response_sha256": "22f3f662b3cd4397890f370e722d6135314b9f7f0f1327fefd5049c24a1c3e03", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "C", + "id": "PUB-C-V09", + "inference_certified": true, + "inference_certified_at_turn": 1, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 1, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 882, + "total_elapsed_ms": 71578.2, + "total_prompt_tokens": 1449, + "total_retrieved_plane1_tokens": 1208, + "total_tokens": 2331, + "turns": [ + { + "candidate_state_hash": "95fd7697e8f45d8796c4aa67", + "completion_tokens": 882, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 100.07, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 71577.99, + "inference_oracle_ms": 0.21, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1449, + "response_sha256": "2f69e2cac346e0b2e0f859d23c79b7f2d1d065261a2c606f216735c6d35dbda4", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + } + ], + "turns_used": 1 + }, + { + "family": "C", + "id": "PUB-C-V10", + "inference_certified": true, + "inference_certified_at_turn": 3, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 3, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "MISSING_WHAT_IS_NOT_PROVED", + "state_hash": "b35119e1ad41", + "turn": 1 + }, + { + "primary_rule": "MISSING_WHAT_IS_NOT_PROVED", + "state_hash": "5adcc26417d9", + "turn": 2 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 2451, + "total_elapsed_ms": 195797.09, + "total_prompt_tokens": 4490, + "total_retrieved_plane1_tokens": 3605, + "total_tokens": 6941, + "turns": [ + { + "candidate_state_hash": "b35119e1ad41a36f135a9385", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "eval_grader_ms": 160.17, + "eval_passed": false, + "failure_signature": "3668bc04ca209bbe827dbfdd", + "generation_ms": 83629.23, + "inference_oracle_ms": 0.22, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1135, + "response_sha256": "0d91e9f4c4bad26c4635e2bf2829e4cf73bf9b9ba424bf32452ce3c3faceef06", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "5adcc26417d974cc4da8086b", + "completion_tokens": 923, + "diagnostic_count": 2, + "diagnostic_rules": [ + "MISSING_WHAT_IS_NOT_PROVED", + "MISSING_WHAT_YOU_MUST_TRUST" + ], + "eval_grader_ms": 206.55, + "eval_passed": false, + "failure_signature": "3668bc04ca209bbe827dbfdd", + "generation_ms": 75611.4, + "inference_oracle_ms": 0.46, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1973, + "response_sha256": "90092e578ca8cf38ca6d12132280307e22524826719ff1e414c93adf77857d58", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "29d346f26ef2727184f57daf", + "completion_tokens": 504, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 167.16, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 36555.61, + "inference_oracle_ms": 0.17, + "inference_passed": true, + "passed": false, + "prompt_tokens": 1382, + "response_sha256": "66ea48e1ac56f677118281732048aeb8a361236e372ee84c18332ab7f763ad98", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_72", + "sec12_78", + "sec7_32", + "sec7_35" + ], + "retrieved_plane1_tokens": 1189, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + } + ], + "turns_used": 3 + }, + { + "family": "C", + "id": "PUB-C-V11", + "inference_certified": true, + "inference_certified_at_turn": 4, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "INVALID_JSON_BLOCK", + "state_hash": "469d8f55abc6", + "turn": 1 + }, + { + "primary_rule": "INVALID_JSON_BLOCK", + "state_hash": "da65ded51134", + "turn": 2 + }, + { + "primary_rule": "INCOMPLETE_TRUST_BOUNDARY", + "state_hash": "31be02ffd918", + "turn": 3 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 4081, + "total_elapsed_ms": 324268.1, + "total_prompt_tokens": 7042, + "total_retrieved_plane1_tokens": 4824, + "total_tokens": 11123, + "turns": [ + { + "candidate_state_hash": "469d8f55abc6a08278854561", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "INVALID_JSON_BLOCK" + ], + "eval_grader_ms": 1.65, + "eval_passed": false, + "failure_signature": "8b5f85706efaa7b5838dea27", + "generation_ms": 85145.24, + "inference_oracle_ms": 0.26, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1371, + "response_sha256": "61ef260ed471bbba1b913ad93d6c8d2a246dc5cac2ae752a9181ad4965b9befb", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "da65ded51134867e99ce960b", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "INVALID_JSON_BLOCK" + ], + "eval_grader_ms": 1.08, + "eval_passed": false, + "failure_signature": "8b5f85706efaa7b5838dea27", + "generation_ms": 87236.41, + "inference_oracle_ms": 0.57, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2039, + "response_sha256": "f122c919445ec6a31429b37f8271ec0788ea4534dfe403e37e5c1c8b71716869", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "31be02ffd9187801bcfb6b04", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 58.41, + "eval_passed": false, + "failure_signature": "8fbfbc3d3fca02c956be297e", + "generation_ms": 72357.52, + "inference_oracle_ms": 62.46, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1609, + "response_sha256": "bc1b9410700383baf6a85b2f6aa01dd3f899ffbcbe417cbe828abdaef805e4a9", + "retrieved_chunk_ids": [ + "sec12_66", + "sec12_69", + "sec12_78", + "sec8_39" + ], + "retrieved_plane1_tokens": 1200, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "dfea6da144e4d8f764e22dd3", + "completion_tokens": 1009, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 2.01, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 79464.59, + "inference_oracle_ms": 1.05, + "inference_passed": true, + "passed": false, + "prompt_tokens": 2023, + "response_sha256": "f992d97bb4c52a938183c43f742ff4e4b85774854a67ae26023d9bf2241e7222", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "C", + "id": "PUB-C-V12", + "inference_certified": true, + "inference_certified_at_turn": 2, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 2, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "INCOMPLETE_TRUST_BOUNDARY", + "state_hash": "1630d9f53870", + "turn": 1 + } + ], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 1938, + "total_elapsed_ms": 162737.5, + "total_prompt_tokens": 3860, + "total_retrieved_plane1_tokens": 2416, + "total_tokens": 5798, + "turns": [ + { + "candidate_state_hash": "1630d9f53870ad569fd1765b", + "completion_tokens": 914, + "diagnostic_count": 1, + "diagnostic_rules": [ + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 79.8, + "eval_passed": false, + "failure_signature": "8fbfbc3d3fca02c956be297e", + "generation_ms": 79930.47, + "inference_oracle_ms": 0.17, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1456, + "response_sha256": "2ce5a3948478e5e94d4e7040a7aecd3c96e9a93f7b6e9c57e50b8b7f87b03ed8", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "491e4952a8aa6a359204f960", + "completion_tokens": 1024, + "diagnostic_count": 0, + "diagnostic_rules": [], + "eval_grader_ms": 91.65, + "eval_passed": false, + "failure_signature": "PASS", + "generation_ms": 82806.21, + "inference_oracle_ms": 0.65, + "inference_passed": true, + "passed": false, + "prompt_tokens": 2404, + "response_sha256": "71cbe8aa15849654bc3b61e9d9784c9d8238ca53975ac4d2fa17dc07a3136614", + "retrieved_chunk_ids": [ + "sec7_35", + "sec8_37" + ], + "retrieved_plane1_tokens": 1208, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + } + ], + "turns_used": 2 + }, + { + "family": "D", + "id": "PUB-D-V01", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "2b597c44d5e4", + "turn": 1 + }, + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "3cd81fc0ba69", + "turn": 2 + }, + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "8c84251a646f", + "turn": 3 + }, + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "d1a5691397e4", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 2893, + "total_elapsed_ms": 234487.51, + "total_prompt_tokens": 7498, + "total_retrieved_plane1_tokens": 4758, + "total_tokens": 10391, + "turns": [ + { + "candidate_state_hash": "2b597c44d5e4a1a514e10be6", + "completion_tokens": 446, + "diagnostic_count": 3, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 441.01, + "eval_passed": false, + "failure_signature": "9c71f2d9c5747751ded21e89", + "generation_ms": 33083.18, + "inference_oracle_ms": 447.83, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1319, + "response_sha256": "e3855eb419577c5b33a4a2c28366511278c47c87be007e3f78d58322f7bff5d8", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "3cd81fc0ba696d5456f4f184", + "completion_tokens": 399, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 482.58, + "eval_passed": false, + "failure_signature": "ce0e210a39e79ab778d2808a", + "generation_ms": 35937.55, + "inference_oracle_ms": 557.06, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2050, + "response_sha256": "5de1e8cd87b5f0826d8307589f28f923a831be0a85e8548c36f27de570aed2dc", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "8c84251a646f6ab671a261fe", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 0.35, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 87172.13, + "inference_oracle_ms": 0.04, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1969, + "response_sha256": "df47e9cc6f48b0d54275444b7552722972aa60dd81cb53703f44505fefc0a115", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "d1a5691397e48fa0d8d3a632", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 1.62, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 77289.69, + "inference_oracle_ms": 0.03, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2160, + "response_sha256": "7b50213e09409028f4c7665c7cd7151355a4f88615f95d7f0649b7363b7b2fab", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V02", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "58b86f8d0554", + "turn": 1 + }, + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "1753cbc4de6e", + "turn": 2 + }, + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "a7f7f921d6ab", + "turn": 3 + }, + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "7a8277affde4", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 3, + "total_completion_tokens": 4096, + "total_elapsed_ms": 320593.29, + "total_prompt_tokens": 6808, + "total_retrieved_plane1_tokens": 4783, + "total_tokens": 10904, + "turns": [ + { + "candidate_state_hash": "58b86f8d0554219550695516", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 1.34, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 81306.28, + "inference_oracle_ms": 0.03, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1293, + "response_sha256": "7e37048f87142dbf96f89151999e8df7f8884f85c2287e563e8d31ebd11e1aac", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "1753cbc4de6ed6386f01f47e", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 0.5, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 87152.71, + "inference_oracle_ms": 0.08, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2309, + "response_sha256": "5c658e749edcef4077ee52d2a3f4181e72563b5015db679877abc6f1944987c6", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "a7f7f921d6ab691249a048d0", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 1.31, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 80347.09, + "inference_oracle_ms": 0.03, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1581, + "response_sha256": "cf8122cc80d7daf62f897162f1f047ee30f6ab523880bc50b597af09cb98b597", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_69", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1192, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "7a8277affde4998ef9b845aa", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 0.33, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 71787.03, + "inference_oracle_ms": 0.04, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1625, + "response_sha256": "a96ef489cfc54f5abb2887c42f679253c2f5355da4f1fe96ae39e88267401330", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_72", + "sec12_74", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V03", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "c41bf0011511", + "turn": 1 + }, + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "58c34868f892", + "turn": 2 + }, + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "90a55a7fd768", + "turn": 3 + }, + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "90a55a7fd768", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 3, + "total_completion_tokens": 4096, + "total_elapsed_ms": 320156.95, + "total_prompt_tokens": 6832, + "total_retrieved_plane1_tokens": 4783, + "total_tokens": 10928, + "turns": [ + { + "candidate_state_hash": "c41bf0011511aa0598159e44", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 0.9, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 82692.99, + "inference_oracle_ms": 0.08, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1318, + "response_sha256": "30411c93dd0052e378b44d18ce354c7edf669276ac949430fe60a0e941f70aec", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "58c34868f892274471d24b96", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 0.71, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 86700.73, + "inference_oracle_ms": 0.05, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2255, + "response_sha256": "707a266e7998a0f57987d1f65fa79ecf43cf0e32cfa4a98ea6ef35dcd853a78c", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "90a55a7fd76859ddb87e391a", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 0.4, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 72114.39, + "inference_oracle_ms": 0.04, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1607, + "response_sha256": "2bebad03602003de125554285e0dd55b9ee7749c8dd189baac229c23ea4b3704", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_69", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1192, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "90a55a7fd76859ddb87e391a", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 0.98, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 78648.56, + "inference_oracle_ms": 0.11, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1652, + "response_sha256": "2bebad03602003de125554285e0dd55b9ee7749c8dd189baac229c23ea4b3704", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_72", + "sec12_74", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V04", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "3bd3cc53d054", + "turn": 1 + }, + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "3bd3cc53d054", + "turn": 2 + }, + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "6934529e5a7c", + "turn": 3 + }, + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "902d3d834a1e", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 3, + "total_completion_tokens": 4096, + "total_elapsed_ms": 303701.08, + "total_prompt_tokens": 6822, + "total_retrieved_plane1_tokens": 4783, + "total_tokens": 10918, + "turns": [ + { + "candidate_state_hash": "3bd3cc53d054fa890411484a", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 1.38, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 73248.78, + "inference_oracle_ms": 0.03, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1334, + "response_sha256": "dd53a8b013c0b6b52f032e36af6674310651c381bab1fcef0d4c25883999f191", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "3bd3cc53d054fa890411484a", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 0.28, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 73572.71, + "inference_oracle_ms": 0.02, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2200, + "response_sha256": "dd53a8b013c0b6b52f032e36af6674310651c381bab1fcef0d4c25883999f191", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "6934529e5a7c850362ffd5fc", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 1.59, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 83969.64, + "inference_oracle_ms": 0.07, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1621, + "response_sha256": "4ae62258d7e66e8247b53a8e4c3b322f7aae1fd46658794e0288890ea2384d3c", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_69", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1192, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "902d3d834a1ea01885930f62", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 0.35, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 72909.79, + "inference_oracle_ms": 0.04, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1667, + "response_sha256": "d9bc4c5dd8ea7f7c03a0ff6893e9b8fb500be322f64a35027473d83ecd32a79e", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_72", + "sec12_74", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V05", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "7f2228450af0", + "turn": 1 + }, + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "b9a415d23ad5", + "turn": 2 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "4f80d1fb419e", + "turn": 3 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "72cbe94abf1c", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 2, + "total_completion_tokens": 2698, + "total_elapsed_ms": 225868.85, + "total_prompt_tokens": 7071, + "total_retrieved_plane1_tokens": 4783, + "total_tokens": 9769, + "turns": [ + { + "candidate_state_hash": "7f2228450af0397a22d881f9", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 1.33, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 81821.92, + "inference_oracle_ms": 0.02, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1302, + "response_sha256": "33ae9c5b6ab87cd435f2825ef8a333558979453717e4b1542096d246eeb1b787", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "b9a415d23ad5b2e463f27bed", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 0.29, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 83538.87, + "inference_oracle_ms": 0.02, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2380, + "response_sha256": "eb92fc284577745ccf6959898ba2b2fbad75c1f2e53663216b78c1216c48332f", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "4f80d1fb419e59dbbec8b797", + "completion_tokens": 320, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 607.73, + "eval_passed": false, + "failure_signature": "f8f5cb1a12c896c3823575d7", + "generation_ms": 29158.02, + "inference_oracle_ms": 507.42, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1590, + "response_sha256": "913a9aeb66be44b1b2d3b94e80f4fe70160c79f89492d05f1ca881a6bcbf396e", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_69", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1192, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "72cbe94abf1c3034194cdb27", + "completion_tokens": 330, + "diagnostic_count": 2, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 626.66, + "eval_passed": false, + "failure_signature": "f8f5cb1a12c896c3823575d7", + "generation_ms": 30271.3, + "inference_oracle_ms": 571.28, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1799, + "response_sha256": "d1956f8ca01d0de52ab66ca9b1ba9be08e265cfe8bb95bd90cf386997a4ec86e", + "retrieved_chunk_ids": [ + "sec10_56", + "sec10_58", + "sec12_66" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V06", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "a183702fbb8b", + "turn": 1 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "1a47e8714c45", + "turn": 2 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "ced9579e10ab", + "turn": 3 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "6be7605165a6", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 4096, + "total_elapsed_ms": 299357.16, + "total_prompt_tokens": 8440, + "total_retrieved_plane1_tokens": 4743, + "total_tokens": 12536, + "turns": [ + { + "candidate_state_hash": "a183702fbb8bb73d6e3062d1", + "completion_tokens": 1024, + "diagnostic_count": 3, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 649.19, + "eval_passed": false, + "failure_signature": "9c71f2d9c5747751ded21e89", + "generation_ms": 70971.95, + "inference_oracle_ms": 811.8, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1324, + "response_sha256": "c4c8414cddb9d43e7906651b17c9280db8d45ba00239d440c6dcfa9f327f5bed", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "1a47e8714c45117182c35b64", + "completion_tokens": 1024, + "diagnostic_count": 3, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY", + "MISSING_CLAIM_TIER_LABEL" + ], + "eval_grader_ms": 517.14, + "eval_passed": false, + "failure_signature": "bf10546eca81466263111dfb", + "generation_ms": 84789.47, + "inference_oracle_ms": 574.95, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2374, + "response_sha256": "6044e4ff90110e1ddc98bff8876e244e898de8102cb292f9e588ca5dbd7285c2", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "ced9579e10ab8799babc72aa", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 599.82, + "eval_passed": false, + "failure_signature": "b2b690fce4357f0edb286840", + "generation_ms": 73334.3, + "inference_oracle_ms": 519.52, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2377, + "response_sha256": "2d41183dda34ecc85c8becbb27e9da161787ee9130515eb62e1fbd4e73c8f4e8", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "6be7605165a6ca912c6b1f4e", + "completion_tokens": 1024, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 443.6, + "eval_passed": false, + "failure_signature": "b2b690fce4357f0edb286840", + "generation_ms": 67889.51, + "inference_oracle_ms": 465.66, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2365, + "response_sha256": "2b43a712698b4ec3aa1c957186703e63212a6a38baa8d1bb5a7c48c4a7e3cc56", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V07", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "6fe41da20b97", + "turn": 1 + }, + { + "primary_rule": "PROSE_GATE_VIOLATION", + "state_hash": "6803d81b8b5c", + "turn": 2 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "04f38de6ac4d", + "turn": 3 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "2f13fe77ce03", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 2, + "total_completion_tokens": 2847, + "total_elapsed_ms": 199260.34, + "total_prompt_tokens": 7548, + "total_retrieved_plane1_tokens": 4768, + "total_tokens": 10395, + "turns": [ + { + "candidate_state_hash": "6fe41da20b970b5833095308", + "completion_tokens": 742, + "diagnostic_count": 5, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 482.22, + "eval_passed": false, + "failure_signature": "b4a07534ac01fe0c912743c7", + "generation_ms": 59132.95, + "inference_oracle_ms": 459.05, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1313, + "response_sha256": "3498ca95ada52b45f45756a8119a0092c9096280730d37db3e03a5e24a7903fc", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "6803d81b8b5ca3673dc9c476", + "completion_tokens": 897, + "diagnostic_count": 5, + "diagnostic_rules": [ + "PROSE_GATE_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 575.79, + "eval_passed": false, + "failure_signature": "b4a07534ac01fe0c912743c7", + "generation_ms": 59179.87, + "inference_oracle_ms": 568.46, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2236, + "response_sha256": "9307aa4aedf0eafab9a13830446a8d3a96e7173e84075e74ae2b06cda1fdd8fe", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "04f38de6ac4d0cfae003c8f3", + "completion_tokens": 609, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "eval_grader_ms": 521.13, + "eval_passed": false, + "failure_signature": "4c3df851138d0cf7779d3e14", + "generation_ms": 39631.54, + "inference_oracle_ms": 610.86, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1814, + "response_sha256": "61a1d691171b1791f35d952d5b109e4e265a7483d4b2988039cc4c44ce1b2049", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_69", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1192, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "2f13fe77ce0381565e60ee37", + "completion_tokens": 599, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "eval_grader_ms": 452.68, + "eval_passed": false, + "failure_signature": "4c3df851138d0cf7779d3e14", + "generation_ms": 39211.35, + "inference_oracle_ms": 466.26, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2185, + "response_sha256": "9c27e570d0599685baf6c333c8c9861bdfb49bd9195b4adab0668f71f6ff0fe0", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V08", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "565d9fbb48d3", + "turn": 1 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "912c125ef718", + "turn": 2 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "912c125ef718", + "turn": 3 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "accaf810e8aa", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 1387, + "total_elapsed_ms": 118164.79, + "total_prompt_tokens": 6722, + "total_retrieved_plane1_tokens": 4754, + "total_tokens": 8109, + "turns": [ + { + "candidate_state_hash": "565d9fbb48d376312841bc2e", + "completion_tokens": 377, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 479.5, + "eval_passed": false, + "failure_signature": "b2b690fce4357f0edb286840", + "generation_ms": 30374.59, + "inference_oracle_ms": 492.35, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1313, + "response_sha256": "52cab65ae9fea016148e7dfecee1db871c1da2ea2c8eddc0770b5e04a0de2b15", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "912c125ef718e0b79faa1ae3", + "completion_tokens": 343, + "diagnostic_count": 1, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 551.02, + "eval_passed": false, + "failure_signature": "c0bd5455dfb6a4bc2799c1a9", + "generation_ms": 27707.98, + "inference_oracle_ms": 527.21, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1905, + "response_sha256": "9d2ce58aada94d898b469d76c13f79eb9496bc51319f003da673e92ce4aade86", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "912c125ef718e0b79faa1ae3", + "completion_tokens": 343, + "diagnostic_count": 1, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 525.75, + "eval_passed": false, + "failure_signature": "c0bd5455dfb6a4bc2799c1a9", + "generation_ms": 33615.93, + "inference_oracle_ms": 708.09, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1836, + "response_sha256": "9d2ce58aada94d898b469d76c13f79eb9496bc51319f003da673e92ce4aade86", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "accaf810e8aa4e783bafd4e1", + "completion_tokens": 324, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "eval_grader_ms": 752.55, + "eval_passed": false, + "failure_signature": "4c3df851138d0cf7779d3e14", + "generation_ms": 24149.23, + "inference_oracle_ms": 589.41, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1668, + "response_sha256": "01bf97989999a7cc95843743630bb386f31e1b539bbc7139ba5e5e0b93b9f51c", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_69", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1193, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V09", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "c85ddaff46d9", + "turn": 1 + }, + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "987182d7a04a", + "turn": 2 + }, + { + "primary_rule": "PROSE_DUPES_VIOLATION", + "state_hash": "915c8c8c39da", + "turn": 3 + }, + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "a20c561ad708", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 0, + "total_completion_tokens": 2696, + "total_elapsed_ms": 209155.96, + "total_prompt_tokens": 7881, + "total_retrieved_plane1_tokens": 4758, + "total_tokens": 10577, + "turns": [ + { + "candidate_state_hash": "c85ddaff46d9ab55e2fee984", + "completion_tokens": 596, + "diagnostic_count": 3, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 496.11, + "eval_passed": false, + "failure_signature": "9c71f2d9c5747751ded21e89", + "generation_ms": 41235.56, + "inference_oracle_ms": 1347.66, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1317, + "response_sha256": "ce32030ac28982129916d61a48bba1b963bba66d2182ac39f7985fdb8a08e5f8", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "987182d7a04a1e26822f5b1e", + "completion_tokens": 643, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 488.85, + "eval_passed": false, + "failure_signature": "ce0e210a39e79ab778d2808a", + "generation_ms": 49661.7, + "inference_oracle_ms": 486.52, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2185, + "response_sha256": "1141604d31a44ab4634c663bd70702afd20899c725c690678086a8ea4fd1a47f", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "915c8c8c39da46bf06cf197b", + "completion_tokens": 851, + "diagnostic_count": 3, + "diagnostic_rules": [ + "PROSE_DUPES_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 598.74, + "eval_passed": false, + "failure_signature": "75fc15322324cfe5451a95c3", + "generation_ms": 65815.67, + "inference_oracle_ms": 551.43, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2203, + "response_sha256": "f595418890597d174831d21dac9b8b74630bfd92d2a3af919114259af2e56169", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "a20c561ad708f8a8fb0e2b10", + "completion_tokens": 606, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 655.79, + "eval_passed": false, + "failure_signature": "ce0e210a39e79ab778d2808a", + "generation_ms": 49391.53, + "inference_oracle_ms": 665.89, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2176, + "response_sha256": "45421dd679a8e6acb9f662e9f72b8db534cc4f2969cbb4dc49498a1b1ebd9687", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V10", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "248d076dc684", + "turn": 1 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "450bf2184a08", + "turn": 2 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "868e0b538512", + "turn": 3 + }, + { + "primary_rule": "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "state_hash": "f8991b324818", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 2, + "total_completion_tokens": 1812, + "total_elapsed_ms": 149921.65, + "total_prompt_tokens": 7060, + "total_retrieved_plane1_tokens": 4754, + "total_tokens": 8872, + "turns": [ + { + "candidate_state_hash": "248d076dc68428ea9e34b24a", + "completion_tokens": 429, + "diagnostic_count": 2, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "INCOMPLETE_TRUST_BOUNDARY" + ], + "eval_grader_ms": 507.53, + "eval_passed": false, + "failure_signature": "b2b690fce4357f0edb286840", + "generation_ms": 36063.07, + "inference_oracle_ms": 511.39, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1330, + "response_sha256": "9d6d1fc18e71bf75c0ba9cf30231ced2d23f86971a340ec447b81e864dc3add3", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "450bf2184a08a1e694b2d7e1", + "completion_tokens": 524, + "diagnostic_count": 1, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 508.64, + "eval_passed": false, + "failure_signature": "c0bd5455dfb6a4bc2799c1a9", + "generation_ms": 39396.31, + "inference_oracle_ms": 2216.3, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1975, + "response_sha256": "fdb2ead17318027051af1d64ac6013d992d4b6a9355eaa334e7399aab4a397d5", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "868e0b5385123e551354e704", + "completion_tokens": 509, + "diagnostic_count": 1, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 481.38, + "eval_passed": false, + "failure_signature": "c0bd5455dfb6a4bc2799c1a9", + "generation_ms": 43537.47, + "inference_oracle_ms": 472.63, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2069, + "response_sha256": "de4e725c23e5db62e7ad574bbc45bd41dc29755a2b3cb9cf4ed66441e65db258", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 3 + }, + { + "candidate_state_hash": "f8991b3248181b9fa305e797", + "completion_tokens": 350, + "diagnostic_count": 1, + "diagnostic_rules": [ + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 536.57, + "eval_passed": false, + "failure_signature": "c0bd5455dfb6a4bc2799c1a9", + "generation_ms": 27214.94, + "inference_oracle_ms": 509.54, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1686, + "response_sha256": "61a32f0e6f20d59cff3057c4b3619377ac605a5ec2837fe15e07725952ddc6d0", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_69", + "sec12_72", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1193, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": true, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V11", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "653dc27b1df2", + "turn": 1 + }, + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "49fa35a94087", + "turn": 2 + }, + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "b1d1b1cb5005", + "turn": 3 + }, + { + "primary_rule": "PROSE_DUPES_VIOLATION", + "state_hash": "e56ca26cbbf5", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 3129, + "total_elapsed_ms": 199253.91, + "total_prompt_tokens": 7383, + "total_retrieved_plane1_tokens": 4768, + "total_tokens": 10512, + "turns": [ + { + "candidate_state_hash": "653dc27b1df28dafbd63e4ae", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 2.0, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 80160.87, + "inference_oracle_ms": 0.08, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1319, + "response_sha256": "3fb90c34f28f8224aa3db9fcd06bb0400629dd7909b42e7c24d2078ac47324bb", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "49fa35a9408729b07b46dfcf", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 0.4, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 64451.04, + "inference_oracle_ms": 0.03, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2400, + "response_sha256": "68899bd108542a656428602f6e5af4c5a3eee95e71adcf77d54c388c92dbdd3d", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "b1d1b1cb50054960d448ecef", + "completion_tokens": 388, + "diagnostic_count": 4, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR", + "MISSING_CLAIM_TIER_LABEL" + ], + "eval_grader_ms": 513.68, + "eval_passed": false, + "failure_signature": "55647d951f2abe4d47d73b96", + "generation_ms": 26713.52, + "inference_oracle_ms": 554.0, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1607, + "response_sha256": "fd3256a2971ee4a13df2c56711911a32ff27ad20e3ad7cb96fb6b95859e5c7e1", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_69", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1192, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "e56ca26cbbf516aee2041e6d", + "completion_tokens": 693, + "diagnostic_count": 4, + "diagnostic_rules": [ + "PROSE_DUPES_VIOLATION", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 424.4, + "eval_passed": false, + "failure_signature": "75fc15322324cfe5451a95c3", + "generation_ms": 26918.72, + "inference_oracle_ms": 455.65, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2057, + "response_sha256": "d5ee12c6ce51e667b38d2b873463fd4c1b01aabe3bfb6e272493a85bf1355032", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + }, + { + "family": "D", + "id": "PUB-D-V12", + "inference_certified": false, + "inference_certified_at_turn": null, + "label_free_inference_stopping": true, + "max_turns": 4, + "oracle_calls": 4, + "pass_at_1": false, + "pass_at_2": false, + "pass_at_3": false, + "pass_at_4": false, + "passed": false, + "refuted_routes": [ + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "ea0bf0cd05f8", + "turn": 1 + }, + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "354fd7a857d0", + "turn": 2 + }, + { + "primary_rule": "EMBEDDED_LEAN_NON_INIT_TACTIC", + "state_hash": "c7cd4017e1fc", + "turn": 3 + }, + { + "primary_rule": "MISSING_EMBEDDED_LEAN_BLOCK", + "state_hash": "ee62ee1db4c6", + "turn": 4 + } + ], + "solved_at_turn": null, + "stagnation_events": 1, + "total_completion_tokens": 3447, + "total_elapsed_ms": 124076.15, + "total_prompt_tokens": 6882, + "total_retrieved_plane1_tokens": 4768, + "total_tokens": 10329, + "turns": [ + { + "candidate_state_hash": "ea0bf0cd05f83492360031fd", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 1.66, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 64565.74, + "inference_oracle_ms": 0.08, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1312, + "response_sha256": "4cfb6866d67bae1433b2ea967d2f8cbae6ea0b2a8ea1702e2f1663e9d2e59a16", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 1 + }, + { + "candidate_state_hash": "354fd7a857d04c0ddcef6214", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 1.78, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 28376.73, + "inference_oracle_ms": 0.12, + "inference_passed": false, + "passed": false, + "prompt_tokens": 2010, + "response_sha256": "aa83f84ea4d97a2e2103781a142a6b65f0b7bf75895ba07161314b7eb1e08362", + "retrieved_chunk_ids": [ + "sec10_58", + "sec7_32" + ], + "retrieved_plane1_tokens": 1197, + "stagnation_detected_after_turn": true, + "stagnation_prompt_mode": false, + "turn": 2 + }, + { + "candidate_state_hash": "c7cd4017e1fcd07b9e47472c", + "completion_tokens": 375, + "diagnostic_count": 3, + "diagnostic_rules": [ + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_NON_INIT_TACTIC", + "EMBEDDED_LEAN_LEAN_COMPILE_ERROR" + ], + "eval_grader_ms": 534.45, + "eval_passed": false, + "failure_signature": "ce0e210a39e79ab778d2808a", + "generation_ms": 8247.03, + "inference_oracle_ms": 492.28, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1599, + "response_sha256": "4f0d1d5e3a0ce08fca469725a1a5b9012ebab939e3b494fce49e83f3327ce304", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_69", + "sec12_78", + "sec9_48" + ], + "retrieved_plane1_tokens": 1192, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": true, + "turn": 3 + }, + { + "candidate_state_hash": "ee62ee1db4c684f051b8bf35", + "completion_tokens": 1024, + "diagnostic_count": 1, + "diagnostic_rules": [ + "MISSING_EMBEDDED_LEAN_BLOCK" + ], + "eval_grader_ms": 0.3, + "eval_passed": false, + "failure_signature": "adfd52b47184abc54d0919aa", + "generation_ms": 22394.13, + "inference_oracle_ms": 0.04, + "inference_passed": false, + "passed": false, + "prompt_tokens": 1961, + "response_sha256": "4d936bc38e9d6591fa4a13568a9f115c4af44298ce848af70342492382ee9012", + "retrieved_chunk_ids": [ + "sec10_60", + "sec12_66", + "sec12_78", + "sec9_44" + ], + "retrieved_plane1_tokens": 1182, + "stagnation_detected_after_turn": false, + "stagnation_prompt_mode": false, + "turn": 4 + } + ], + "turns_used": 4 + } + ], + "label_free_inference_stopping": true, + "max_turns": 4, + "pass_curve": { + "k=1": { + "elapsed_ms_up_to_k": 3290749.52, + "pass_rate_pct": 0.0, + "passed": 0, + "replies_used_up_to_k": 48, + "tokens_used_up_to_k": 108368, + "total": 48 + }, + "k=2": { + "elapsed_ms_up_to_k": 6452996.33, + "pass_rate_pct": 0.0, + "passed": 0, + "replies_used_up_to_k": 95, + "tokens_used_up_to_k": 249509, + "total": 48 + }, + "k=3": { + "elapsed_ms_up_to_k": 9303315.8, + "pass_rate_pct": 0.0, + "passed": 0, + "replies_used_up_to_k": 138, + "tokens_used_up_to_k": 358909, + "total": 48 + }, + "k=4": { + "elapsed_ms_up_to_k": 11826729.84, + "pass_rate_pct": 0.0, + "passed": 0, + "replies_used_up_to_k": 176, + "tokens_used_up_to_k": 466971, + "total": 48 + } + }, + "preservation_probes_8": { + "details": [ + { + "id": "pres_1", + "passed": true, + "reply_preview": "391" + }, + { + "id": "pres_2", + "passed": true, + "reply_preview": "500" + }, + { + "id": "pres_3", + "passed": true, + "reply_preview": "1024" + }, + { + "id": "pres_4", + "passed": true, + "reply_preview": "Tokyo" + }, + { + "id": "pres_5", + "passed": true, + "reply_preview": "No" + }, + { + "id": "pres_6", + "passed": true, + "reply_preview": "len" + }, + { + "id": "pres_7", + "passed": true, + "reply_preview": "404" + }, + { + "id": "pres_8", + "passed": true, + "reply_preview": "Yes" + } + ], + "passed": 8, + "total": 8 + }, + "retrieval_max_tokens": 1250, + "schema": "mcproof-repair-loop-receipt/v0.2", + "split": "validation", + "total_items": 48 +} diff --git a/compile/artifacts/evaluation/repair_loop/validation_repair_summary.json b/compile/artifacts/evaluation/repair_loop/validation_repair_summary.json new file mode 100644 index 0000000..63eda4b --- /dev/null +++ b/compile/artifacts/evaluation/repair_loop/validation_repair_summary.json @@ -0,0 +1,268 @@ +{ + "arms": { + "B0": { + "budget_totals": { + "completion_tokens": 98510, + "inference_certified_items": 23, + "mean_retrieved_plane1_tokens_per_turn": 1194.8, + "oracle_calls": 139, + "prompt_tokens": 241755, + "replies_used": 139, + "stagnation_events": 38, + "total_tokens": 340265 + }, + "by_family": { + "A": { + "k=1": { + "passed": 1, + "total": 12 + }, + "k=2": { + "passed": 1, + "total": 12 + }, + "k=3": { + "passed": 1, + "total": 12 + }, + "k=4": { + "passed": 1, + "total": 12 + } + }, + "B": { + "k=1": { + "passed": 4, + "total": 12 + }, + "k=2": { + "passed": 5, + "total": 12 + }, + "k=3": { + "passed": 5, + "total": 12 + }, + "k=4": { + "passed": 5, + "total": 12 + } + }, + "C": { + "k=1": { + "passed": 0, + "total": 12 + }, + "k=2": { + "passed": 0, + "total": 12 + }, + "k=3": { + "passed": 0, + "total": 12 + }, + "k=4": { + "passed": 0, + "total": 12 + } + }, + "D": { + "k=1": { + "passed": 0, + "total": 12 + }, + "k=2": { + "passed": 0, + "total": 12 + }, + "k=3": { + "passed": 0, + "total": 12 + }, + "k=4": { + "passed": 2, + "total": 12 + } + } + }, + "pass_curve": { + "k=1": { + "elapsed_ms_up_to_k": 2640926.5, + "pass_rate_pct": 10.42, + "passed": 5, + "replies_used_up_to_k": 48, + "tokens_used_up_to_k": 98485, + "total": 48 + }, + "k=2": { + "elapsed_ms_up_to_k": 4454668.95, + "pass_rate_pct": 12.5, + "passed": 6, + "replies_used_up_to_k": 81, + "tokens_used_up_to_k": 190318, + "total": 48 + }, + "k=3": { + "elapsed_ms_up_to_k": 6237362.22, + "pass_rate_pct": 12.5, + "passed": 6, + "replies_used_up_to_k": 111, + "tokens_used_up_to_k": 269185, + "total": 48 + }, + "k=4": { + "elapsed_ms_up_to_k": 7869835.55, + "pass_rate_pct": 16.67, + "passed": 8, + "replies_used_up_to_k": 139, + "tokens_used_up_to_k": 340265, + "total": 48 + } + }, + "preservation_probes_8": { + "passed": 8, + "total": 8 + } + }, + "P2-B": { + "budget_totals": { + "completion_tokens": 157295, + "inference_certified_items": 14, + "mean_retrieved_plane1_tokens_per_turn": 1195.7, + "oracle_calls": 176, + "prompt_tokens": 309676, + "replies_used": 176, + "stagnation_events": 67, + "total_tokens": 466971 + }, + "by_family": { + "A": { + "k=1": { + "passed": 0, + "total": 12 + }, + "k=2": { + "passed": 0, + "total": 12 + }, + "k=3": { + "passed": 0, + "total": 12 + }, + "k=4": { + "passed": 0, + "total": 12 + } + }, + "B": { + "k=1": { + "passed": 0, + "total": 12 + }, + "k=2": { + "passed": 0, + "total": 12 + }, + "k=3": { + "passed": 0, + "total": 12 + }, + "k=4": { + "passed": 0, + "total": 12 + } + }, + "C": { + "k=1": { + "passed": 0, + "total": 12 + }, + "k=2": { + "passed": 0, + "total": 12 + }, + "k=3": { + "passed": 0, + "total": 12 + }, + "k=4": { + "passed": 0, + "total": 12 + } + }, + "D": { + "k=1": { + "passed": 0, + "total": 12 + }, + "k=2": { + "passed": 0, + "total": 12 + }, + "k=3": { + "passed": 0, + "total": 12 + }, + "k=4": { + "passed": 0, + "total": 12 + } + } + }, + "pass_curve": { + "k=1": { + "elapsed_ms_up_to_k": 3290749.52, + "pass_rate_pct": 0.0, + "passed": 0, + "replies_used_up_to_k": 48, + "tokens_used_up_to_k": 108368, + "total": 48 + }, + "k=2": { + "elapsed_ms_up_to_k": 6452996.33, + "pass_rate_pct": 0.0, + "passed": 0, + "replies_used_up_to_k": 95, + "tokens_used_up_to_k": 249509, + "total": 48 + }, + "k=3": { + "elapsed_ms_up_to_k": 9303315.8, + "pass_rate_pct": 0.0, + "passed": 0, + "replies_used_up_to_k": 138, + "tokens_used_up_to_k": 358909, + "total": 48 + }, + "k=4": { + "elapsed_ms_up_to_k": 11826729.84, + "pass_rate_pct": 0.0, + "passed": 0, + "replies_used_up_to_k": 176, + "tokens_used_up_to_k": 466971, + "total": 48 + } + }, + "preservation_probes_8": { + "passed": 8, + "total": 8 + } + } + }, + "base_gguf_path": "/compile/checkpoints/Qwen3.5-4B-GGUF/Qwen3.5-4B-Q8_0.gguf", + "base_gguf_sha256": "10cc391b403021dd11c614679d2fd92f611c3681d29e29651b717316965d61e1", + "decoding": { + "per_turn_slot_erase": true, + "seed": 42, + "temperature": 0.0, + "top_p": 1.0 + }, + "llama_server_live_n_ctx": 6144, + "llama_server_version": "version: 0.5.0 (build 11146, commit 7fe450e19)", + "mcproof_v02_git_head": "155a474b5270a7771017a62f8e1db71eb2b75fe5", + "p2b_gguf_path": "/compile/artifacts/p2b/formal-verification.p2b.f16.gguf", + "p2b_gguf_sha256": "728272c0974b795ceb990fde1e215b62d64e6faf04fb799c0fd3e01fa4539ba5", + "plane1_spec_sha256": "49beff2a0185cfa4032170b614a543c0d67541494e858803d21cfb837111b230", + "schema": "mcproof-issue5-validation-summary/v0.2", + "validation_jsonl_sha256": "7e403727c8c02bd75c319f85042dc6a1bc6166c731681644237e4eaa3a4ae621" +} diff --git a/compile/artifacts/p2b/v01_plane1_snapshot/lean/Mcproof.lean b/compile/artifacts/p2b/v01_plane1_snapshot/lean/Mcproof.lean new file mode 100644 index 0000000..d2a7613 --- /dev/null +++ b/compile/artifacts/p2b/v01_plane1_snapshot/lean/Mcproof.lean @@ -0,0 +1,34 @@ +/- + Core verified definitions for the mcproof Lean 4 reference package. + Uses stock Lean 4 Init only, with zero floats and zero escape hatches. +-/ + +namespace Mcproof + +/-- Move `x` into the closed natural-number interval `[lo, hi]`. -/ +def clampNat (lo hi x : Nat) : Nat := + if x < lo then lo + else if hi < x then hi + else x + +/-- Check exact integer Bezout identity `a * x + b * y = g`. -/ +def bezoutCheck (a b x y g : Int) : Bool := + decide (a * x + b * y = g) + +/-- Division-free rational order check for `n1 / d1 <= n2 / d2` with positive denominators. -/ +def ratLeCross (n1 d1 n2 d2 : Int) : Bool := + decide (0 < d1 ∧ 0 < d2 ∧ n1 * d2 ≤ n2 * d1) + +/-- True when a single path segment is non-empty, not `.` or `..`, and free of `/` and NUL. -/ +def isSafeSegment (seg : String) : Bool := + decide (seg ≠ "" ∧ seg ≠ "." ∧ seg ≠ ".." ∧ ¬ seg.contains '/' ∧ ¬ seg.contains '\x00') + +/-- Validate a pre-tokenized list of relative path segments with no traversal or embedded slash. -/ +def cleanSegments (segs : List String) : Option (List String) := + match segs with + | [] => none + | s :: rest => + if (s :: rest).all isSafeSegment then some (s :: rest) + else none + +end Mcproof diff --git a/compile/artifacts/p2b/v01_plane1_snapshot/lean/Mcproof/Proofs.lean b/compile/artifacts/p2b/v01_plane1_snapshot/lean/Mcproof/Proofs.lean new file mode 100644 index 0000000..fb02256 --- /dev/null +++ b/compile/artifacts/p2b/v01_plane1_snapshot/lean/Mcproof/Proofs.lean @@ -0,0 +1,74 @@ +/- + Complete proofs of all theorems in `Mcproof/Spec.lean`. + Zero `sorry`, zero custom axioms, and zero compiler escape hatches. +-/ + +import Mcproof + +open Mcproof + +theorem Mcproof.Spec.clampNat_in_range (lo hi x : Nat) (h : lo ≤ hi) : + lo ≤ clampNat lo hi x ∧ clampNat lo hi x ≤ hi := by + unfold clampNat + split + · omega + · split <;> omega + +theorem Mcproof.Spec.clampNat_of_mem (lo hi x : Nat) (h1 : lo ≤ x) (h2 : x ≤ hi) : + clampNat lo hi x = x := by + unfold clampNat + split + · omega + · split <;> omega + +theorem Mcproof.Spec.bezoutCheck_sound (a b x y g : Int) : + bezoutCheck a b x y g = true → a * x + b * y = g := by + unfold bezoutCheck + intro h + exact of_decide_eq_true h + +theorem Mcproof.Spec.bezoutCheck_complete (a b x y g : Int) : + a * x + b * y = g → bezoutCheck a b x y g = true := by + unfold bezoutCheck + intro h + exact decide_eq_true h + +theorem Mcproof.Spec.ratLeCross_sound (n1 d1 n2 d2 : Int) : + ratLeCross n1 d1 n2 d2 = true → 0 < d1 ∧ 0 < d2 ∧ n1 * d2 ≤ n2 * d1 := by + unfold ratLeCross + intro h + exact of_decide_eq_true h + +theorem Mcproof.Spec.ratLeCross_refl_pos (n d : Int) (hd : 0 < d) : + ratLeCross n d n d = true := by + unfold ratLeCross + apply decide_eq_true + exact ⟨hd, hd, Int.le_refl (n * d)⟩ + +theorem Mcproof.Spec.cleanSegments_safe (segs out : List String) : + cleanSegments segs = some out → + out ≠ [] ∧ ∀ s ∈ out, s ≠ "" ∧ s ≠ "." ∧ s ≠ ".." ∧ ¬ s.contains '/' ∧ ¬ s.contains '\x00' := by + intro h + cases segs with + | nil => + simp [cleanSegments] at h + | cons hd tl => + change (if (hd :: tl).all isSafeSegment = true then some (hd :: tl) else none) = some out at h + by_cases hall : (hd :: tl).all isSafeSegment = true + · simp [hall] at h + subst h + refine ⟨by simp, ?_⟩ + intro s hs + have hseg : isSafeSegment s = true := List.all_eq_true.mp hall s hs + unfold isSafeSegment at hseg + exact of_decide_eq_true hseg + · simp [hall] at h + +theorem Mcproof.Spec.cleanSegments_singleton (s : String) + (hs : s ≠ "" ∧ s ≠ "." ∧ s ≠ ".." ∧ ¬ s.contains '/' ∧ ¬ s.contains '\x00') : + cleanSegments [s] = some [s] := by + unfold cleanSegments + have hseg : isSafeSegment s = true := by + unfold isSafeSegment + exact decide_eq_true hs + simp [hseg] diff --git a/compile/artifacts/p2b/v01_plane1_snapshot/lean/Mcproof/Spec.lean b/compile/artifacts/p2b/v01_plane1_snapshot/lean/Mcproof/Spec.lean new file mode 100644 index 0000000..dee23b6 --- /dev/null +++ b/compile/artifacts/p2b/v01_plane1_snapshot/lean/Mcproof/Spec.lean @@ -0,0 +1,43 @@ +/- + Challenge specification for the mcproof Lean 4 reference package. + Each safety theorem is paired with a usefulness theorem so a trivial + constant or rejecting implementation fails `lake comparator --paranoid`. +-/ + +import Mcproof + +open Mcproof + +/-- Safety: when `lo <= hi`, `clampNat lo hi x` lies inside `[lo, hi]`. -/ +theorem Mcproof.Spec.clampNat_in_range (lo hi x : Nat) (h : lo ≤ hi) : + lo ≤ clampNat lo hi x ∧ clampNat lo hi x ≤ hi := sorry + +/-- Usefulness: an in-range input is returned unchanged. -/ +theorem Mcproof.Spec.clampNat_of_mem (lo hi x : Nat) (h1 : lo ≤ x) (h2 : x ≤ hi) : + clampNat lo hi x = x := sorry + +/-- Soundness: `bezoutCheck` returns `true` only when `a * x + b * y = g`. -/ +theorem Mcproof.Spec.bezoutCheck_sound (a b x y g : Int) : + bezoutCheck a b x y g = true → a * x + b * y = g := sorry + +/-- Completeness: `bezoutCheck` accepts every valid integer witness. -/ +theorem Mcproof.Spec.bezoutCheck_complete (a b x y g : Int) : + a * x + b * y = g → bezoutCheck a b x y g = true := sorry + +/-- Soundness: `ratLeCross` certifies positive denominators and exact cross-multiplied order. -/ +theorem Mcproof.Spec.ratLeCross_sound (n1 d1 n2 d2 : Int) : + ratLeCross n1 d1 n2 d2 = true → 0 < d1 ∧ 0 < d2 ∧ n1 * d2 ≤ n2 * d1 := sorry + +/-- Usefulness: every fraction with positive denominator is less than or equal to itself. -/ +theorem Mcproof.Spec.ratLeCross_refl_pos (n d : Int) (hd : 0 < d) : + ratLeCross n d n d = true := sorry + +/-- Safety: every segment accepted by `cleanSegments` is non-empty and free of traversal or `/`. -/ +theorem Mcproof.Spec.cleanSegments_safe (segs out : List String) : + cleanSegments segs = some out → + out ≠ [] ∧ ∀ s ∈ out, s ≠ "" ∧ s ≠ "." ∧ s ≠ ".." ∧ ¬ s.contains '/' ∧ ¬ s.contains '\x00' := sorry + +/-- Usefulness: a single safe segment is accepted unchanged (`none`-always implementations fail). -/ +theorem Mcproof.Spec.cleanSegments_singleton (s : String) + (hs : s ≠ "" ∧ s ≠ "." ∧ s ≠ ".." ∧ ¬ s.contains '/' ∧ ¬ s.contains '\x00') : + cleanSegments [s] = some [s] := sorry diff --git a/compile/artifacts/p2b/v01_plane1_snapshot/lean/comparator.json b/compile/artifacts/p2b/v01_plane1_snapshot/lean/comparator.json new file mode 100644 index 0000000..655ce43 --- /dev/null +++ b/compile/artifacts/p2b/v01_plane1_snapshot/lean/comparator.json @@ -0,0 +1,19 @@ +{ + "challenge_module": "Mcproof.Spec", + "solution_module": "Mcproof.Proofs", + "theorem_names": [ + "Mcproof.Spec.clampNat_in_range", + "Mcproof.Spec.clampNat_of_mem", + "Mcproof.Spec.bezoutCheck_sound", + "Mcproof.Spec.bezoutCheck_complete", + "Mcproof.Spec.ratLeCross_sound", + "Mcproof.Spec.ratLeCross_refl_pos", + "Mcproof.Spec.cleanSegments_safe", + "Mcproof.Spec.cleanSegments_singleton" + ], + "permitted_axioms": [ + "propext", + "Quot.sound", + "Classical.choice" + ] +} diff --git a/compile/artifacts/p2b/v01_plane1_snapshot/lean/lakefile.toml b/compile/artifacts/p2b/v01_plane1_snapshot/lean/lakefile.toml new file mode 100644 index 0000000..4e243f7 --- /dev/null +++ b/compile/artifacts/p2b/v01_plane1_snapshot/lean/lakefile.toml @@ -0,0 +1,18 @@ +name = "mcproof" +version = "0.1.0" +defaultTargets = ["Mcproof", "McproofSpec", "McproofProofs", "mcproof_lean"] + +[[lean_lib]] +name = "Mcproof" + +[[lean_lib]] +name = "McproofSpec" +roots = ["Mcproof.Spec"] + +[[lean_lib]] +name = "McproofProofs" +roots = ["Mcproof.Proofs"] + +[[lean_exe]] +name = "mcproof_lean" +root = "Main" diff --git a/compile/artifacts/p2b/v01_plane1_snapshot/spec/lineage.json b/compile/artifacts/p2b/v01_plane1_snapshot/spec/lineage.json new file mode 100644 index 0000000..97d9250 --- /dev/null +++ b/compile/artifacts/p2b/v01_plane1_snapshot/spec/lineage.json @@ -0,0 +1,139 @@ +{ + "_meta": { + "gg.siliconcordat/mde": { + "adapter_class": "class1_rederivable", + "hard_vetoes_enforced": [ + 1, + 2, + 3, + 4, + 5 + ], + "l1_bank_tag": 3, + "primary_zone": "formal_proof", + "zone_manifest_id": "sha256:4e00656cb47fce97f687ae638afba22c569b9680b3780eeb2e5ea71a62c55eca" + } + }, + "domain": "formal-verification", + "plane1_spec_bytes": 106763, + "plane1_spec_path": "spec/plane1_formal_spec.md", + "plane1_spec_sha256": "49beff2a0185cfa4032170b614a543c0d67541494e858803d21cfb837111b230", + "plane1_spec_token_stats": { + "char_div4_estimate": 26690, + "cl100k_base_tokens": 30873, + "utf8_bytes": 106763, + "words": 15791 + }, + "schema": "mcproof-lineage/v0.1", + "siliconcordat_commits": { + "siliconcordat/exo": "9d52aa1f28272753ddf83dbe02864d05989b74d2", + "siliconcordat/mde": "52ce0178854a67490eb718544961ba6cff710f22", + "siliconcordat/warp": "28f56549a8bb59b723281afa31831df0824e7dec", + "siliconcordat/weft": "3d66f07e5ee67863a7e6c1029e7473c9c8c11b08" + }, + "source_lineage": { + "contested_heads": [], + "corpus_tree_sha256": "5bdb7f30363b956e82947f668f98c48f99876e87fb5ed4f785b873e4f4f98305", + "domain_commits": { + "corpus": "7b22e542f0db0f3013ab1439cf5d1947676a5083", + "formal-verification": "49beff2a0185cfa4032170b614a543c0d6754149", + "weft-eido": "be27665255d9aa2553060d275f7d1b1b3306c703" + } + }, + "tracked_plane1_files": { + "lean/Mcproof.lean": "e27f69bce4f77b2b0fc308880a84e8e6c0710b578532167f1e27905595f0722c", + "lean/Mcproof/Proofs.lean": "27d1ae954c2e3ed1b76df409e0f59f41eca9c1ab13e6f4263d9a88a2ef8c6731", + "lean/Mcproof/Spec.lean": "0e4f403806727a088382477a3eddf8f7d67c0377a3e7391af862f7da1a67cca5", + "lean/comparator.json": "223b808dc8266557829fa21d515a0b0bbdf3f28a7242b03a23c1d806520d34e8", + "lean/lakefile.toml": "8540e8392738181588c16176e164496608dc5c89e1d359150d991d6f18c3d0ec", + "lean/lean-toolchain": "ca5f1f601d34c49079e38d7e7041eb0f2a9c227fbc1df0f404820069a052330c", + "schemas/dataset-record.v0.1.schema.json": "5891b56778196ac7d9a2ea30c5d673c2f7a3f5daa1da0da34c91518a482cbbea", + "schemas/lineage.v0.1.schema.json": "1872cc74c04ca505ab6de39a197a77a839423384da0f6059f2e61d3a937341fb", + "schemas/oracle-receipt.v0.1.schema.json": "9a1758658537c7d6d1c14e6cc1b6680eb7eddebe9e59c9c35255a839d0c53f80", + "schemas/zone-manifest.v0.1.json": "496ab13ac619e4dce959ae23c0f4ff06c8b74afb36869b0a226b9b8e9d0dcc1d", + "spec/plane1_formal_spec.md": "49beff2a0185cfa4032170b614a543c0d67541494e858803d21cfb837111b230" + }, + "upstream_repos": { + "Foundation-0/.github": { + "commit": "81f83c38918382b4ca142b81debd3cc5ccf61d3a", + "roles": [ + "REPO-STANDARD.md" + ], + "url": "https://github.com/Foundation-0/.github" + }, + "Foundation-0/f0": { + "commit": "9a37134750174339c53121939838ce62ab8af630", + "roles": [ + "scripts/gate.py", + "tests/run_gate_corruptions.py" + ], + "url": "https://github.com/Foundation-0/f0" + }, + "Foundation-0/finite-state-proofs": { + "commit": "9eeb3da2cdd6ba6b100ac24d23b32c369c0db8ec", + "roles": [ + "verify.py" + ], + "url": "https://github.com/Foundation-0/finite-state-proofs" + }, + "Foundation-0/formal-claims-audit": { + "commit": "bacf05953c10bf5105fa1209a127392d395e2f92", + "roles": [ + "scripts/check_verdicts.py" + ], + "url": "https://github.com/Foundation-0/formal-claims-audit" + }, + "Foundation-0/incentre-lattices": { + "commit": "2d374fff2a59cc9331566733043a4bb25c1040bf", + "roles": [ + "README.md", + "docs/audit.md" + ], + "url": "https://github.com/Foundation-0/incentre-lattices" + }, + "Foundation-0/proven-commons": { + "commit": "903c1858be856776de21675cf5c4e26fb9eebcdf", + "roles": [ + "README.md", + "scripts/audit.py" + ], + "url": "https://github.com/Foundation-0/proven-commons" + }, + "Moissanite-Labs/siliquant": { + "commit": "67b29a4e18cd5d93e1f3701a34030f2e089e7312", + "roles": [ + "Phase 3 loud-column outlier-protection & noise-imatrix reference" + ], + "url": "https://github.com/Moissanite-Labs/siliquant" + }, + "Moissanite-Labs/weft-eido": { + "commit": "be27665255d9aa2553060d275f7d1b1b3306c703", + "roles": [ + "Multi-MDE Phase 6 WCAG contrast & UI module reference" + ], + "url": "https://github.com/Moissanite-Labs/weft-eido" + }, + "Proven-Commons/mission": { + "commit": "7b22e542f0db0f3013ab1439cf5d1947676a5083", + "roles": [ + "playbook/verified-bar.md", + "bar/check.py", + "bar/reviewers.toml", + "reviews/tar/evidence/WeakStatement.lean" + ], + "url": "https://github.com/Proven-Commons/mission" + }, + "ahrazzle/soko": { + "commit": "3165e20e7a16c0035cab2fd01231e78e4e1d7f86", + "roles": [ + "soko_mcp/lean4_auditor.py", + "soko_mcp/checker_auditor.py", + "soko_mcp/integrity.py", + "soko_mcp/evidence.py", + "docs/verified-systems-loop.md" + ], + "url": "https://github.com/ahrazzle/soko" + } + }, + "zone_manifest_id": "sha256:4e00656cb47fce97f687ae638afba22c569b9680b3780eeb2e5ea71a62c55eca" +} diff --git a/compile/src/mcproof_compile/hardware_probe.py b/compile/src/mcproof_compile/hardware_probe.py index 1c45cfb..feba518 100644 --- a/compile/src/mcproof_compile/hardware_probe.py +++ b/compile/src/mcproof_compile/hardware_probe.py @@ -331,37 +331,47 @@ def compute_rtx3080_vram_fit( gguf_bytes: int, *, ctx_tokens: int = 8192, + n_slots: int = 1, n_loras: int = 3, lora_rank: int = 8, d_in: int = 2560, d_out: int = 9216, + windows_wddm_desktop_gib: float = 1.20, ) -> dict[str, Any]: """Analytical VRAM projection for Qwen3.5-4B on an 8 GiB RTX 3080 Laptop GPU (not live NVML). Qwen3.5-4B has 32 layers total: 8 full-attention GQA layers (4 KV heads, head_dim=256) - and 24 Gated DeltaNet linear-attention layers (fixed recurrent state independent of ctx). + and 24 Gated DeltaNet linear-attention layers (fixed per-slot recurrent state independent of ctx). + Under `-np 8 -c 65536` (`kv_unified=false`), KV and DeltaNet state scale with `n_slots = 8`, + while NVIDIA Optimus (`Intel iGPU` driving panels) yields `windows_wddm_desktop_gib = 0.0`. """ + slots = max(1, int(n_slots)) gib = float(1024**3) weights_gib = gguf_bytes / gib - kv_f16_bytes = 8 * 2 * 4 * 256 * ctx_tokens * 2 - ssm_state_bytes = 24 * (32 * 128 * 128 * 4 + 9216 * 4 * 4) + kv_per_slot_bytes = 8 * 2 * 4 * 256 * ctx_tokens * 2 + ssm_per_slot_bytes = 24 * (32 * 128 * 128 * 4 + 9216 * 4 * 4) + kv_f16_bytes = slots * kv_per_slot_bytes + ssm_state_bytes = slots * ssm_per_slot_bytes lora_bytes = n_loras * (d_out * lora_rank + lora_rank * d_in) * 2 - cuda_compute_overhead_gib = 0.55 - windows_wddm_desktop_gib = 1.20 + cuda_compute_overhead_gib = 0.30 if slots >= 8 else 0.55 total_gib = weights_gib + (kv_f16_bytes + ssm_state_bytes + lora_bytes) / gib + cuda_compute_overhead_gib - total_windows_gib = total_gib + windows_wddm_desktop_gib + total_windows_gib = total_gib + float(windows_wddm_desktop_gib) + np8_optimus_gib = weights_gib + (8 * (kv_per_slot_bytes + ssm_per_slot_bytes) + lora_bytes) / gib + 0.30 return { "measurement_kind": "analytical_projection_not_live_nvml", + "n_slots": slots, "model_weights_gib": round(weights_gib, 4), "kv_cache_8k_f16_gib": round(kv_f16_bytes / gib, 4), "deltanet_recurrent_state_gib": round(ssm_state_bytes / gib, 4), "three_f16_rank8_loras_gib": round(lora_bytes / gib, 6), "cuda_context_and_compute_overhead_gib": cuda_compute_overhead_gib, "estimated_total_vram_gib": round(total_gib, 4), - "windows_wddm_desktop_overhead_gib": windows_wddm_desktop_gib, + "estimated_np8_optimus_zero_display_vram_gib": round(np8_optimus_gib, 4), + "windows_wddm_desktop_overhead_gib": float(windows_wddm_desktop_gib), "estimated_total_vram_windows_wddm_gib": round(total_windows_gib, 4), "rtx3080_budget_gib": 7.2, "fits_8gib_rtx3080_budget": total_gib <= 7.2, + "fits_8gib_rtx3080_np8_optimus_budget": np8_optimus_gib <= 7.2, "fits_8gib_rtx3080_windows_wddm_budget": total_windows_gib <= 7.2, } diff --git a/docs/CONTINUITY.md b/docs/CONTINUITY.md index fb6f5c0..5fda783 100644 --- a/docs/CONTINUITY.md +++ b/docs/CONTINUITY.md @@ -21,7 +21,7 @@ 1. **Zero opaque weight blobs:** Every `Class 1` slab, GGUF LoRA, certificate, binding, gate policy, Warp bundle, and W-MCP metadata file (`9/9` `P2-B` files) must rebuild with `0` byte differences on CPU `float64` from pinned Plane 1 files, `Hybrid K0`, and base checkpoint weights (`verification/independent_rederive_p2b.py`). 2. **Zero silent conflict resolution:** Contested syntax zones (`html_delivery`) raise `ContestedPreferenceError` at L4 `exo` and `ERR_CONTESTED_LINEAGE` at L3 `warp-modeld`, dropping active adapter scales to `0.0`. -3. **Zero stale lineage:** Any `1`-byte edit to the `11` tracked Plane 1 files invalidates `corpus_tree_sha256` (`5bdb7f30363b956e82947f668f98c48f99876e87fb5ed4f785b873e4f4f98305`) and triggers automatic adapter unload (`ERR_STALE_LINEAGE`). +3. **Zero stale lineage:** Any `1`-byte edit to the `13` tracked Plane 1 files invalidates `corpus_tree_sha256` (`93624aae...` in v0.2, vs frozen v0.1 snapshot `5bdb7f30...`) and triggers automatic adapter unload (`ERR_STALE_LINEAGE`). 4. **Zero adapter-supplied canaries:** Inline or forged canary fields inside an adapter bundle fail closed (`ERR_FORGED_CANARY_REJECTED`), and `ExternalCommandCanaryRunner` rejects aggregate-only JSON lacking per-canary `outcomes` (`>= min_preservation`). 5. **Zero gate-lowering on Honest Kill Boundary:** Failing public validation (`2/48 < B0 4/48`), primary private holdout (`0/96`), or live `T-C1` `step0` composition retention keeps deep proof search in Plane 1 retrieval and Tier 2 compiler loops without lowering any threshold. @@ -31,7 +31,7 @@ | --- | --- | --- | | L5 `siliconcordat/mde` (`52ce017`) | Canonical `zone-manifest.v0.1.json` (`sha256:4e00656c...`), CST span tagging, `SHA256SUMS` integrity, `guard --code-audit` | Verified directly against `schemas/zone-manifest.v0.1.json`, `datasets/SHA256SUMS` (`312953bc...`), and repository `.py`/`.json` files | | L4 `siliconcordat/exo` | `ZoneRouter` routing `formal_proof` to `formal-verification` (`l1_bank_tag = 3`), `ContestedPreferenceError` on `html_delivery`, `data-eido-skip` audit | [siliconcordat/exo PR four](https://github.com/siliconcordat/exo/tree/feat/mde4-phase4-phase6-stack) (`e68c406`, `40/40` tests, mirrored in `stack/patches/exo-mde4-phase4-phase6-stack.patch`) | -| L3 `siliconcordat/warp` | Four-stage admission gate (`verify_stage1`, fixed `PRE_REGISTERED_GATE_POLICY`, per-canary opaque Stage 3 outcomes, `< 10 ms` multi-slot `O(1)` kill switch) | [siliconcordat/warp PR 5](https://github.com/siliconcordat/warp/pull/5) (`390176c`, `57/57` tests, mirrored in `stack/patches/warp-mde4-phase4-live-gate.patch`) | +| L3 `siliconcordat/warp` | Four-stage admission gate (`verify_stage1`, fixed `PRE_REGISTERED_GATE_POLICY`, per-canary opaque Stage 3 outcomes, `< 10 ms` multi-slot `O(1)` kill switch) | [siliconcordat/warp live-gate branch](https://github.com/siliconcordat/warp/tree/feat/mde4-phase4-live-gate) (`390176c`, `57/57` tests, mirrored in `stack/patches/warp-mde4-phase4-live-gate.patch`) | | L2 `siliconcordat/weft` (`3d66f07`) | CPU `float64` `BetaEditNullspaceProjector`, `FixedRadixCertificate`, `weft-slab/v0.1`, deterministic F16 GGUF v3 LoRA writer | Cross-checked by zero-`weft`-import pure-NumPy rebuild across all `9/9` `P2-B` outputs (`verification/p2b_rederivation_receipt.json`) | | Companion `Moissanite-Labs/weft-eido` | Phase 6 UI contrast audit across `360` OKLCH hues (`0` dark-theme WCAG failures) | Tracked in `stack/patches/weft-eido-pr25-contrast-audit.patch` (`d87b958`, `120/120` tests) | | Companion `Moissanite-Labs/siliquant` | Loud-column activation-weighted quantization ablation (`ffn_down` probe and compiled `P2-B` `ffn_up`) | [Moissanite-Labs/siliquant PR 12](https://github.com/Moissanite-Labs/siliquant/pull/12) (`67b29a4`, `2.295x` on compiled `P2-B` `ffn_up`, `20.766x` on seeded Gaussian `ffn_down` null-space probe) | @@ -79,9 +79,15 @@ python3 verification/build_final_receipt.py 1. **Null-space projection prevents catastrophic forgetting (`85.0%` vs `0.0%`), while single-layer rank-8 steering regresses on public validation (`2/48` vs `B0` `4/48`):** At matched Frobenius norm (`4.336`), unprojected `P2-A` leaked `34,545x` more `K0` energy than `P2-B` (`2.01e13` vs `5.83e8` scaled) and collapsed to `0/100` on `general_retention_100.jsonl` (`P2-B` and zone-routed 3-pack both scored `85/100` vs `B0` `84/100`). However, on public `validation.jsonl` (`48` items), `P2-B` (`2/48`) trailed bare `B0` (`4/48`), `goodhart` (`4/48`), constant `logit_bias` (`6/48`), and `B1` (`8/48`). 2. **Single-pass `4B` proof synthesis is capacity-bound (`0/96` across `B0`, `P2-B`, and `B1`):** Injecting the full `32,216`-token Plane 1 spec (`B1`, `2.05x` slower, `b1_prefix_cache_parity_verified = true` across `3` live full-string items) also scored `0/96` on `private_holdout.jsonl`. 3. **Asymmetric `int4/int8` leakage gain depends strongly on target matrix geometry (`2.295x` on `P2-B` `ffn_up` vs `20.766x` on `ffn_down` probe):** On compiled `P2-B` (`blk.31.ffn_up.weight`, `9216 x 2560`), `int4/int8` (`58,357` B, `69.2%` smaller than F16) reduced `K0` Frobenius leakage `2.295x` vs `int4/int4` (`top-10` column protection `0.9998x`). The larger `20.766x` (`0.8797x` with top-10 imatrix protection) occurs on a seeded Gaussian null-space probe on `ffn_down` (`2560 x 9216`), where base INT4 activation-weighted imatrix protection cuts `K0` output drift `10.27%` while static weight-norm top-10 shifts drift `-0.01%`. -4. **CPU linear subspace orthogonality (`99.85%`) does not imply live nonlinear `step0` composition invariance (`T-C1` FAIL):** When `weft-eido` and `corpus` co-activate at `[0.2, 1.0, 0.0]` (`ui_module`), CPU linear subspace alignment stays `>= 99.85%`, yet nonlinear transformer propagation shifts `weft-eido` live `step0` logprobs by `0.0611` (`31.77x` its `0.0019` solo shift, `0.0%` live retention). Gate `T-C1` must measure live `step0` drift directly without `max()` fallback to CPU subspace scores. +4. **CPU linear subspace orthogonality (`99.85%`) does not imply live nonlinear `step0` composition invariance (`T-C1` FAIL), resolved by SwiGLU + RMSNorm tangent multi-layer projection:** + - When `weft-eido` and `corpus` co-activate at `[0.2, 1.0, 0.0]` (`ui_module`), CPU linear subspace alignment stays `>= 99.85%`, yet elementwise `silu(W_gate x)` breaks `X X^T` orthogonality and single-layer residual edits induce `-0.5 * (||delta||^2 / ||h||^2) * h` radial shrinkage (`0.0611` shift vs `0.0019` solo, `0.0%` retention). + - Seven `6`-kernel Lean `v4.35.0` theorems (`Mcproof.AdapterBounds`) and 4-layer `Qwen3.5-4B` `blk.28..31` `TangentBetaEdit` (`12/12` `0`-byte re-derivation) cut `up_proj` leakage `29,456x`, cut per-layer RMSNorm shrinkage `8.58x`, and lift live `llama-server` `[1.0, 1.0, 1.0]` `T-C1` retention from `0.00%` to `90.73%` (`PASS >= 90.0%`, `24/24` anchors, `95.21%` at `[0.2, 1.0, 0.0]`), while preventing `` token attenuation under `enable_thinking: false`. +5. **Live 8 GiB RTX 3080 `T-C5` hardware replay (`UX582HS`, `driver 566.07`, `CUDA 12.7`, `-c 65536 -np 8`) passed across both `Q8_0` and `Q4_K_M`:** `Qwen3.5-4B-Q8_0.gguf` + 3 F16 LoRAs peaked at `6.8525 GiB <= 7.2 GiB` (`1`-slot `+0.38%`, `8`-slot `-0.66%` overhead) and `Q4_K_M` peaked at `5.2314 GiB` (`+0.47%` / `+0.48%` overhead, `stack/results/final/ux582hs_rtx3080_tc5_live_receipt.json`). Multi-slot `8 x 8,192` KV takes `~2.0 GiB` while NVIDIA Optimus WDDM desktop reserve is `0 MiB` because Intel UHD drives both panels (`hardware_probe.py::compute_rtx3080_vram_fit` updated with `n_slots`). +6. **Label-free `k = 1, 2, 4` Plane 1 retrieval + Tier 2 repair (`mcproof` v0.2, [Proven-Commons/mcproof issue 5](https://github.com/Proven-Commons/mcproof/issues/5)) doubles public `B0` pass rate (`4/48 -> 8/48`) and breaks the `0/96` sealed holdout ceiling:** + - Public `validation.jsonl` (`48` items): `B0` climbed from `5/48` (`10.42%`, `k=1`) to `6/48` (`12.50%`, `k=2`) to `8/48` (`16.67%`, `k=4`), while single-layer `blk.31` `P2-B` scored `0/48` (`67` stagnation events). + - Sealed `private_holdout.jsonl` (`96` items, `sealed_holdout_repair_loop_receipt.json`): `B0` reached `1/96` (`HOLD-C-04` at `k=2..4`, `77` stagnation events) and `P2-B` reached `1/96` (`HOLD-D-10` at `k=3..4`, `85` stagnation events) with `0` vault leaks. ## Evidence Limits and Next Actions -1. **Live 8 GiB RTX 3080 workbench telemetry replay (`geminus`):** Live serving ran on a 64 GiB Apple M1 Max (`llama-server 0.5.0`, `$0` cloud) while `geminus` (`UX582HS`, 8 GiB RTX 3080) had SSH and RPC ports closed (`linux_rtx3080_online = false`, `vram_measurement_kind = analytical_projection_not_nvml`). Replay `T-C5` `nvidia-smi` telemetry once Linux workbench SSH is provisioned. -2. **Multi-turn Plane 1 retrieval and Tier 2 repair loop harness:** Build a bounded multi-turn Plane 1 section-retrieval + Tier 2 `mcproof grade` diagnostic feedback loop (`k <= 4` compiler rounds) and benchmark pass@k across `validation.jsonl` and `private_holdout.jsonl` under unchanged Tier 2 oracles. +1. **Multi-layer `blk.28..31` `TangentBetaEdit` live serving promotion:** Wire 4-layer `TangentBetaEdit` (`weft` `c33bbf4f`) into live `warp-modeld` Stage 1 to 4 admission and measure multi-turn `k = 1..4` pass curves with `blk.28..31` non-uniform depth allocation (`alpha_l*` proportional to `R_l^2 / (gamma_l^2 * rho_l)`). +2. **Concrete graph-hypothesis coupling for `Mcproof.TransitionBounds`:** Mechanize finite-table checkers discharging projection and cut-capacity hypotheses directly inside Lean `v4.35.0` alongside Python `verify_dual_product_parity`. diff --git a/docs/CROSS_POLLINATION.md b/docs/CROSS_POLLINATION.md new file mode 100644 index 0000000..41c5f75 --- /dev/null +++ b/docs/CROSS_POLLINATION.md @@ -0,0 +1,56 @@ +# Bidirectional Cross-Pollination Ledger (`mcproof` v0.2, `soko`, and Collaborator Repos) + +This ledger records every formal theorem, certificate gate, and closed-loop repair mechanism transferred between `Proven-Commons/mcproof`, `ahrazzle/soko`, and collaborator formal-verification repositories. It pairs exact file paths, symbol names, and pull requests across repositories so verifiers can audit each dependency without guessing. Engineers extending `mde:formal-verification` or `mde:soko-eval` use this page to keep domain-general kernels separate from specialist search harnesses. + +## Architectural Boundary + +`Proven-Commons/mcproof` owns the domain-general Formal Verification MDE (`mde:formal-verification`), including the stock-`Init` Lean `v4.35.0` kernel library, label-free Plane 1 retrieval, multi-turn repair loop, and `F-01` through `F-08` certificate auditors (`https://github.com/Proven-Commons/mcproof/issues/5`). `ahrazzle/soko` owns the specialist state-space search harness (`mde:soko-eval`), dual transition engines, and frontier lower-bound certificates (`https://github.com/ahrazzle/soko/issues/53`). + +## Inbound Transfers (`ahrazzle/soko` to `Proven-Commons/mcproof`) + +Every failure mode and solver pattern discovered during `ahrazzle/soko` frontier audits was abstracted into a domain-general `mcproof` module or `6`-kernel Lean `v4.35.0` theorem with zero domain-specific code in `mcproof`. + +| `soko` Source (`repo::path::symbol` or PR) | Abstracted `mcproof` Target (`path::symbol`) | Mechanism Transferred | Verification Evidence | +| --- | --- | --- | --- | +| `soko::soko_mcp/lean4_auditor.py` and `checker_auditor.py` | `src/mcproof/oracles.py::run_lean4_oracle` and `run_checker_oracle` | Escape-hatch rejection, vacuous-hypothesis (`WeakStatement`) hunting, exact `int`/`Fraction` AST enforcement, and dual-skeleton hashing | `100%` tamper rejection (`10/10`) and `>= 80%` planted-bug kill floor | +| `soko::soko_mcp/formal_kernel.py` (`https://github.com/ahrazzle/soko/pull/58`) | `src/mcproof/domain_protocol.py::audit_transition_certificate`, `verify_dual_product_parity`, `audit_portfolio_bounds` | Fail-closed detectors for `F-01` single-oracle flags, `F-02` partial-state hashes, `F-03` unlocked TCB pins, `F-04` unwitnessed floors, `F-05` truncated potentials, `F-06` coarse-observation `(spec, impl)` drift, `F-07` caller bound inflation, and `F-08` unregistered digests | `18/18` corruption and dual-product vectors rejected fail-closed in `tests/test_domain_and_repair_loop.py` | +| `soko::soko_mcp/evidence.py::verify_certificate_gate` (`https://github.com/ahrazzle/soko/pull/60`) | `src/mcproof/domain_protocol.py::audit_transition_certificate` and `src/mcproof/oracles.py::run_checker_oracle` | `A6` non-empty, distinct, non-tautological `must_fail_controls` enforcement (`oracle_invoked=True`) and `A7` `LINT_PASS` / `LOWER_BOUND` tier-swap rejection | Static zero-case audits report `LINT_PASS` and block promotion to `VERIFIED` or `PROVEN` | +| `soko::docs/verified-systems-loop.md` and `https://github.com/ahrazzle/soko/issues/38` | `src/mcproof/repair_loop.py::compute_candidate_state_hash`, `compute_failure_signature`, `run_repair_item` | AST/token state-hash deduplication and `(rule, line)` plateau detection under `temperature = 0.0`, rotating Plane 1 chunks toward worked examples and dropping repeated broken drafts | Stagnating Turn 2 ASTs trigger `Stagnation Escape Alert` and Plane 1 chunk rotation under `<= 1,500` tokens | +| `soko` TCB catalog discipline (`https://github.com/ahrazzle/soko/pull/57`) and C++20 9-flow verifier (`https://github.com/ahrazzle/soko/pull/59`) | `lean/Mcproof/TransitionBounds.lean` and `src/mcproof/lineage.py::verify_lineage_manifest` | Decoupled TCB `SHA256SUMS` pinning plus pure discrete state-transition, operator-counting LP dual, cyclic mode-switch, conservation-harvest, and vertex-split min-cut bounds | `29/29` theorems verified by `6/6` kernels (`lean-paranoid`, `lean4lean`, `nanoda`, `con-leche`, `con-ron`, `lean-default`) and `370` differential cases | +| `soko::mde_soko/` `S3` empirical trace telemetry (`https://github.com/ahrazzle/soko/pull/62`) | `src/mcproof/domain_protocol.py::verify_minimal_goal_stopping` | Rejects synthesized action traces that continue emitting actions after reaching a goal state (`CERT_POST_GOAL_EXTRA_ACTIONS`) | Unit-tested in `tests/test_domain_and_repair_loop.py` | + +## Inbound Collaborator Transfers (`1337isnot1337`, `rohanarun`, `CrocSwap`, `eumemic` to `mcproof`) + +Active mathematical and formal-verification repositories across collaborators contributed concrete kernel-isolation, exact-JSON, and repair-loop safeguards in `mcproof` v0.2. + +| Collaborator Source (`repo::path::symbol`) | `mcproof` Target (`path::symbol`) | Mechanism Transferred | Verification Evidence | +| --- | --- | --- | --- | +| `1337isnot1337/verifier::scripts/inside.py` and `scripts/verdicts.py` | `src/mcproof/lean_harness.py::scrub_challenge_artifacts`, `classify_host_resource_failure`, and `repair_loop.py::sanitize_oracle_diagnostics` | Unlinks compiled `Spec.*` `.olean`/`.ir` before rebuild, splits `INCOMPLETE_HOST_LIMIT` (OOM/timeout) from mathematical `REJECTED`, and scrubs `@`-mentions and fences from compiler diagnostics | Unit-tested in `tests/test_lean_harness.py` and `tests/test_domain_and_repair_loop.py` | +| `CrocSwap/lean-buildtools::lean_buildtools/process.py` and `CrocSwap/CertiFFI::tools/check-directive.sh` | `src/mcproof/lean_harness.py::build_capped_lean_args`, `verify_pinned_lean_defs`, and `verify_lean4_against_challenge` | Enforces `--trust=0 -j 1 -M 1400 -D Elab.async=false`, blocks pattern-matching `def` and `abbrev` rewrites (`TARGET_DEF_BODY_MUTATED`), and checks `Meta.isDefEq` kernel theorem types against prompt and challenge specs | Rejects theorem-statement substitutions, comment-only mentions, and mutated `def`/`abbrev` bodies | +| `CrocSwap/integer-mult-bounds::scripts/verify_rational_lp.py` | `src/mcproof/domain_protocol.py::loads_strict_certificate_json` and `verify_rational_lp_certificate` | Parse-time JSON rejection of floats, `NaN`/`Infinity`, duplicate keys, `bool` subclasses of `int`, and `> 10^60` integers across primal/dual `bounds` and Farkas `infeasible` LP certificates | Verified in `mcproof audit-cert` CLI and `verify_rational_lp_certificate` unit tests | +| `rohanarun/yang-mills-mass-gap-auto-research::CriterionAudit.lean` and `negative-results/` | `src/mcproof/lean_harness.py::audit_weak_statements` and `src/mcproof/repair_loop.py::run_repair_item` | Detects circular binder premises matching the theorem conclusion (`VACUOUS_TAUTOLOGICAL_HYPOTHESIS`) and records `refuted_routes` (`state_hash`, `primary_rule`) so repair turns skip refuted proof paths | Verified on circular `0 < gap` premise and `Refuted Routes (Do Not Re-Explore)` prompt state | +| `rohanarun/emeans-plus::check_claims.py`, `rohanarun/quasi-rh-hillclimb::check_controls.py`, and `eumemic/stdhttp-verify` | `src/mcproof/mutation.py::run_python_strict` and `tests/` | Pairs counterexample repair against non-adversarial retention anchors and enforces `python3 -O -B` assertion-stripping safety across every gate | Full `pytest` suite passes under `python3 -O -B` with `0` assertion-dependent gates | + +## Outbound Transfers (`Proven-Commons/mcproof` to `ahrazzle/soko`) + +`mcproof` exports machine-checked Lean `v4.35.0` admissibility theorems, the pluggable `DomainOracle` protocol, restricted micro-computation sandboxes, and `siliconcordat/mde` code-hygiene gates directly back into `ahrazzle/soko`. + +| `mcproof` Source (`path::symbol`) | `soko` Consumer (`repo::path` or branch) | Mechanism Transferred | Verification Evidence | +| --- | --- | --- | --- | +| `lean/Mcproof/Spec.lean` (`quotientHeuristic_*`, `prefundedHeuristic_*`, `conservation_harvest_potential_repair`) | `soko` frontier quotient and pre-funded harvest certificates (`https://github.com/ahrazzle/soko/pull/61` and `feat/iii14-joint-border-switch-floor`) | Kernel-checked universal `1`-step consistency and goal admissibility for quotient homomorphisms (`hQ(pi(s))`) and `Nat` pre-funded monotone resource potentials (`h(s) - c * r(s)`) | Certified `15x7` `III-14 >= 42` across `741.24M` lossless `Packed24` A* states and `26.42B` quotient edges with `0` Bellman violations | +| `lean/Mcproof/AdapterBounds.lean` (`rmsnorm_*`, `multilayer_*`, `swiglu_*`, `scalar_projection_retention_illusion`) | `siliconcordat/weft` multi-layer `TangentBetaEdit` (`weft/tangent.py`) | Seven `6`-kernel constructive `[propext, Quot.sound]` Lean `v4.35.0` theorems proving RMSNorm tangent orthogonality, `1/L` quadratic radial-shrinkage reduction, SwiGLU `diag(g^2)` / Hadamard adjoint metrics, and rational shrinkage bounds | 4-layer `Qwen3.5-4B` `blk.28..31` `TangentBetaEdit` (`12/12` `0`-byte re-derivation) cuts `up_proj` leakage `29,456x` and lifts live `llama-server` `[1.0, 1.0, 1.0]` `T-C1` `0.00%` to `90.73%` (`95.21%` at `[0.2, 1.0, 0.0]`) | +| `lean/Mcproof/Spec.lean` (`operatorCounting_dual_*`, `dualFloorNat_*`, `ceilDiv_step_consistent_and_admissible`) | `soko` exact rational Farkas / landmark LP dual certificates (`feat/landmark-lp-dual-bounds`) | Division-free integer-scaled dual objective bounds (`dualObjective cs / scale <= planSum cost acts`) and exact rational ceiling `1`-step consistency (`ceilDivNat`) | Certifies fractional landmark dual floors and ceilings in stock `Init` with zero `Float` arithmetic | +| `lean/Mcproof/Spec.lean` (`cyclicSwitchDist_metric_and_witness`, `vertexSplit_unit_flow_horizon_bound`, `zeroSelfLoop_deficit_composition`) | `soko` multi-mode switch, time-expanded vertex-split flow, and partitioned deficit lower bounds | Directed `k`-mode cyclic triangle inequality, unit vertex-capacity horizon min-cut bound (`demand <= capPerStep * T`), and zero-self-loop quotient + deficit composition | Proves all `29` reference theorems across `4,233` kernel declarations (`4,230` in `lean4lean`) with `0` escape hatches | +| `src/mcproof/domain_protocol.py::DomainOracle`, `BudgetLedger`, `run_restricted_sandbox_cell` | `soko::mde_soko/` (`mde:soko-eval` specialist package) | Structural `DomainOracle` protocol, `4.0`-second network-free / filesystem-isolated Python micro-computation cell, and explicit turn/token/step `BudgetLedger` | Specialist domain harnesses plug into `mcproof` repair runners without game code entering `mcproof` | +| `src/mcproof/repair_loop.py::default_inference_oracle` vs `default_dataset_item_grader` | `mcproof` and `soko` closed-loop `k = 1, 2, 4` evaluation harnesses | Strict separation between `100%` label-free inference stopping (`{'id', 'family', 'prompt'}` only) and post-hoc ground-truth benchmark grading (`eval_grader_fn`) | Zero `expected_tier`, `expected_verdict`, or private `valid_cases` leakage into repair prompts | +| `src/mcproof/oracles.py::run_mde_code_audit` (`https://github.com/siliconcordat/mde/issues/4`, `https://github.com/Proven-Commons/mcproof/pull/3`) | `soko` PRs `57`, `58`, `59`, `60` and `mde_soko/` | Tree-sitter `mde guard --code-audit` silent-exception rejection, `python3 -O -B` assertion safety, and `check-prose.py --dupes` enforcement | `0` code-audit or prose-gate violations across all audited `mcproof` and `soko` branches | + +## What is not proved + +- `Mcproof.TransitionBounds` proves soundness and `1`-step consistency conditional on a domain's transition projection, action dual load, or cut-capacity hypotheses. Verifying that a concrete C++, Rust, or Python transition table satisfies those hypotheses requires complete finite-graph enumeration or a mechanized transition-semantics embedding. +- Label-free inference stopping (`default_inference_oracle`) certifies syntax, kernel type-checking, static exact-math rules, and prompt-visible claim hygiene, whereas hidden test-case completeness on Family `B` is measured post-hoc by `eval_grader_fn`. + +## What you must trust + +- Lean `v4.35.0` and all five external kernels configured in `lean/comparator.json` (`leanchecker`, `lean4lean`, `nanoda`, `con-leche`, `con-ron`). +- Subprocess process-group isolation, stripped environment variables, and AST import guards inside `run_restricted_sandbox_cell`, together with SHA-256 manifest verification in `mcproof lineage --verify`. diff --git a/lean/Main.lean b/lean/Main.lean index 16e248d..809f251 100644 --- a/lean/Main.lean +++ b/lean/Main.lean @@ -1,6 +1,8 @@ import Mcproof open Mcproof +open Mcproof.TransitionBounds +open Mcproof.AdapterBounds private def parseInt? (s : String) : Option Int := s.toInt? @@ -8,6 +10,21 @@ private def parseInt? (s : String) : Option Int := private def parseNat? (s : String) : Option Nat := s.toNat? +private def parseNatList? : List String → Option (List Nat) + | [] => some [] + | x :: xs => + match parseNat? x, parseNatList? xs with + | some v, some rest => some (v :: rest) + | _, _ => none + +private def parsePairs? : List String → Option (List (Nat × Nat)) + | [] => some [] + | [_] => none + | wStr :: dStr :: rest => + match parseNat? wStr, parseNat? dStr, parsePairs? rest with + | some w, some d, some tail => some ((w, d) :: tail) + | _, _, _ => none + def main (args : List String) : IO UInt32 := do match args with | ["clamp", loStr, hiStr, xStr] => @@ -42,6 +59,55 @@ def main (args : List String) : IO UInt32 := do | none => IO.println "REJECT" return 0 + | ["prefund", hStr, cStr, rStr] => + match parseNat? hStr, parseNat? cStr, parseNat? rStr with + | some hVal, some cVal, some rVal => + IO.println s!"{prefundedVal hVal cVal rVal}" + return 0 + | _, _, _ => + IO.eprintln "invalid natural numbers" + return 2 + | "dualfloor" :: scaleStr :: pairArgs => + match parseNat? scaleStr, parsePairs? pairArgs with + | some scale, some pairs => + IO.println s!"{dualFloorNat scale pairs}" + return 0 + | _, _ => + IO.eprintln "invalid dualfloor arguments" + return 2 + | ["cyclicdist", kStr, aStr, bStr] => + match parseNat? kStr, parseNat? aStr, parseNat? bStr with + | some k, some a, some b => + IO.println s!"{cyclicSwitchDist k a b}" + return 0 + | _, _, _ => + IO.eprintln "invalid cyclicdist arguments" + return 2 + | ["ceildiv", numStr, scaleStr] => + match parseNat? numStr, parseNat? scaleStr with + | some num, some scale => + IO.println s!"{ceilDivNat num scale}" + return 0 + | _, _ => + IO.eprintln "invalid ceildiv arguments" + return 2 + | ["tangentproj", h1Str, h2Str, d1Str, d2Str] => + match parseInt? h1Str, parseInt? h2Str, parseInt? d1Str, parseInt? d2Str with + | some h1, some h2, some d1, some d2 => + let out := tangentProjScaled [h1, h2] [d1, d2] + IO.println (String.intercalate "," (out.map toString)) + return 0 + | _, _, _, _ => + IO.eprintln "invalid tangentproj arguments" + return 2 + | "sumsq" :: natArgs => + match parseNatList? natArgs with + | some ds => + IO.println s!"{sumSqNat ds}" + return 0 + | none => + IO.eprintln "invalid sumsq arguments" + return 2 | _ => - IO.eprintln "usage: mcproof_lean (clamp|bezout|ratle|segments) ..." + IO.eprintln "usage: mcproof_lean (clamp|bezout|ratle|segments|prefund|dualfloor|cyclicdist|ceildiv|tangentproj|sumsq) ..." return 2 diff --git a/lean/Mcproof.lean b/lean/Mcproof.lean index d2a7613..8cbb960 100644 --- a/lean/Mcproof.lean +++ b/lean/Mcproof.lean @@ -3,6 +3,9 @@ Uses stock Lean 4 Init only, with zero floats and zero escape hatches. -/ +import Mcproof.TransitionBounds +import Mcproof.AdapterBounds + namespace Mcproof /-- Move `x` into the closed natural-number interval `[lo, hi]`. -/ diff --git a/lean/Mcproof/AdapterBounds.lean b/lean/Mcproof/AdapterBounds.lean new file mode 100644 index 0000000..ad4161a --- /dev/null +++ b/lean/Mcproof/AdapterBounds.lean @@ -0,0 +1,66 @@ +/- + Domain-general integer/rational RMSNorm Tangent Projection, Multi-Layer 1/L + Quadratic Radial-Shrinkage Reduction, Orthogonal Adapter Stack Norm Inflation, + SwiGLU Elementwise Gate Metric, Radial-Null Adjoint Orthogonality, Scalar-Projection + Subspace-Retention Illusion, and Rational Shrinkage Sandwich Kernels (mcproof v0.2). + Stock Lean 4.35.0 Init only: zero floats, zero custom axioms, zero escape hatches. +-/ + +namespace Mcproof.AdapterBounds + +/-- Exact integer inner product on aligned lists (`0` on mismatched tails). -/ +def vecDot : List Int → List Int → Int + | [], _ => 0 + | _, [] => 0 + | x :: xs, y :: ys => x * y + vecDot xs ys + +/-- Exact squared Euclidean norm `||v||^2 = ` in `Int`. -/ +def vecNormSq (v : List Int) : Int := + vecDot v v + +/-- Pointwise vector addition in `Int`. -/ +def vecAdd : List Int → List Int → List Int + | [], _ => [] + | _, [] => [] + | x :: xs, y :: ys => (x + y) :: vecAdd xs ys + +/-- Pointwise vector subtraction in `Int`. -/ +def vecSub : List Int → List Int → List Int + | [], _ => [] + | _, [] => [] + | x :: xs, y :: ys => (x - y) :: vecSub xs ys + +/-- Scalar multiplication `c * v` in `Int`. -/ +def vecScale (c : Int) : List Int → List Int + | [] => [] + | x :: xs => (c * x) :: vecScale c xs + +/-- Elementwise (Hadamard / SwiGLU gate) product `g ⊙ v` in `Int`. -/ +def vecHadamard : List Int → List Int → List Int + | [], _ => [] + | _, [] => [] + | g :: gs, v :: vs => (g * v) :: vecHadamard gs vs + +/-- Elementwise square `g ⊙ g` (gate metric diagonal `diag(g^2)`). -/ +def vecSqDiag (g : List Int) : List Int := + vecHadamard g g + +/-- Division-free RMSNorm tangent projection `||h||^2 * delta - * h`. -/ +def tangentProjScaled (h delta : List Int) : List Int := + vecSub (vecScale (vecNormSq h) delta) (vecScale (vecDot h delta) h) + +/-- Sum of natural numbers in a list (total adapter impulse across layers). -/ +def sumNat : List Nat → Nat + | [] => 0 + | x :: xs => x + sumNat xs + +/-- Sum of squared natural numbers in a list (cumulative quadratic radial shrinkage). -/ +def sumSqNat : List Nat → Nat + | [] => 0 + | x :: xs => x * x + sumSqNat xs + +/-- Exact cross-multiplied rational shrinkage sandwich `D2 / (2*(R2 + D2)) <= (qd - qn)/qd <= D2 / (2*R2)`. -/ +def shrinkageSandwichCheck (R2 D2 qn qd : Int) : Bool := + decide (D2 * qd ≤ 2 * (R2 + D2) * (qd - qn) ∧ 2 * R2 * (qd - qn) ≤ D2 * qd) + +end Mcproof.AdapterBounds diff --git a/lean/Mcproof/Proofs.lean b/lean/Mcproof/Proofs.lean index fb02256..a8c56a8 100644 --- a/lean/Mcproof/Proofs.lean +++ b/lean/Mcproof/Proofs.lean @@ -1,11 +1,13 @@ /- - Complete proofs of all theorems in `Mcproof/Spec.lean`. + Complete proofs of all theorems in `Mcproof/Spec.lean` (v0.2). Zero `sorry`, zero custom axioms, and zero compiler escape hatches. -/ import Mcproof open Mcproof +open Mcproof.TransitionBounds +open Mcproof.AdapterBounds theorem Mcproof.Spec.clampNat_in_range (lo hi x : Nat) (h : lo ≤ hi) : lo ≤ clampNat lo hi x ∧ clampNat lo hi x ≤ hi := by @@ -72,3 +74,783 @@ theorem Mcproof.Spec.cleanSegments_singleton (s : String) unfold isSafeSegment exact decide_eq_true hs simp [hseg] + +theorem Mcproof.Spec.quotientHeuristic_step_consistent + {S A Q : Type} (step : S → A → Option S) (pi : S → Q) (qEdge : Q → Q → Prop) (hQ : Q → Nat) + (hproj : ∀ s a s', step s a = some s' → pi s = pi s' ∨ qEdge (pi s) (pi s')) + (hq : ∀ u v, qEdge u v → hQ u ≤ hQ v + 1) : + ∀ s a s', step s a = some s' → quotientHeuristic pi hQ s ≤ quotientHeuristic pi hQ s' + 1 := by + intro s a s' hstep + unfold quotientHeuristic + cases hproj s a s' hstep with + | inl heq => + rw [heq] + omega + | inr hedge => + exact hq (pi s) (pi s') hedge + +theorem Mcproof.Spec.quotientHeuristic_admissible + {S A Q : Type} (step : S → A → Option S) (isGoal : S → Prop) + (pi : S → Q) (qEdge : Q → Q → Prop) (hQ : Q → Nat) + (hproj : ∀ s a s', step s a = some s' → pi s = pi s' ∨ qEdge (pi s) (pi s')) + (hq : ∀ u v, qEdge u v → hQ u ≤ hQ v + 1) + (hgoal : ∀ g, isGoal g → hQ (pi g) = 0) : + ∀ s acts g, ValidPath step s acts g → isGoal g → quotientHeuristic pi hQ s ≤ acts.length := by + intro s acts g hpath + induction hpath with + | nil s0 => + intro hg + unfold quotientHeuristic + have hz : hQ (pi s0) = 0 := hgoal s0 hg + simp [hz] + | @cons s0 s1 g0 a as hstep _ ih => + intro hg + have hcons : quotientHeuristic pi hQ s0 ≤ quotientHeuristic pi hQ s1 + 1 := + Mcproof.Spec.quotientHeuristic_step_consistent step pi qEdge hQ hproj hq s0 a s1 hstep + have htail : quotientHeuristic pi hQ s1 ≤ as.length := ih hg + simp only [List.length_cons] + omega + +theorem Mcproof.Spec.prefundedHeuristic_admissible_and_goal + {S : Type} (h : S → Nat) (c : Nat) (r : S → Nat) : + (∀ s, prefundedHeuristic h c r s ≤ h s) ∧ + (∀ g, h g = 0 → prefundedHeuristic h c r g = 0) ∧ + (∀ s len, h s ≤ len → prefundedHeuristic h c r s ≤ len) := by + refine ⟨?_, ?_, ?_⟩ + · intro s + unfold prefundedHeuristic prefundedVal + omega + · intro g hg + unfold prefundedHeuristic prefundedVal + omega + · intro s len hle + unfold prefundedHeuristic prefundedVal + omega + +theorem Mcproof.Spec.prefundedHeuristic_step_consistent + {S : Type} (h : S → Nat) (c : Nat) (r : S → Nat) (s s' : S) + (hmono : r s' ≤ r s) + (hdrop : h s ≤ h s' + 1 + c * (r s - r s')) : + prefundedHeuristic h c r s ≤ prefundedHeuristic h c r s' + 1 := by + unfold prefundedHeuristic prefundedVal + have hmul_le : c * r s' ≤ c * r s := Nat.mul_le_mul_left c hmono + have hdist : c * (r s - r s') = c * r s - c * r s' := Nat.mul_sub_left_distrib c (r s) (r s') + rw [hdist] at hdrop + omega + +theorem Mcproof.Spec.prefundedVal_bounded_drop_witness : + prefundedVal 12 1 1 = 11 ∧ + prefundedVal 10 1 0 = 10 ∧ + prefundedVal 12 1 1 ≤ prefundedVal 10 1 0 + 1 := by + unfold prefundedVal + omega + +private theorem planSum_add {A : Type} (f g : A → Nat) (acts : List A) : + planSum (fun a => f a + g a) acts = planSum f acts + planSum g acts := by + induction acts with + | nil => rfl + | cons a as ih => + simp only [planSum, ih] + omega + +private theorem planSum_mul_left {A : Type} (w : Nat) (f : A → Nat) (acts : List A) : + w * planSum f acts = planSum (fun a => w * f a) acts := by + induction acts with + | nil => simp [planSum] + | cons a as ih => + simp only [planSum, Nat.mul_add, ih] + +private theorem planSum_mono {A : Type} {f g : A → Nat} (acts : List A) + (hfg : ∀ a, f a ≤ g a) : + planSum f acts ≤ planSum g acts := by + induction acts with + | nil => simp [planSum] + | cons a as ih => + simp only [planSum] + exact Nat.add_le_add (hfg a) ih + +private theorem dualObjective_le_actionDualLoad_sum + {A : Type} (cs : List (DualConstraint A)) (acts : List A) + (hlandmarks : ∀ c ∈ cs, c.demand ≤ planSum c.coeff acts) : + dualObjective cs ≤ planSum (actionDualLoad cs) acts := by + induction cs with + | nil => + simp [dualObjective, actionDualLoad] + | cons c rest ih => + have hc : c.demand ≤ planSum c.coeff acts := hlandmarks c (by simp) + have hwc : c.weight * c.demand ≤ c.weight * planSum c.coeff acts := + Nat.mul_le_mul_left c.weight hc + have hrest : dualObjective rest ≤ planSum (actionDualLoad rest) acts := by + apply ih + intro c' hc' + exact hlandmarks c' (by simp [hc']) + have hsum : + c.weight * planSum c.coeff acts + planSum (actionDualLoad rest) acts = + planSum (actionDualLoad (c :: rest)) acts := by + rw [planSum_mul_left] + exact (planSum_add (fun a => c.weight * c.coeff a) (actionDualLoad rest) acts).symm + unfold dualObjective + calc + c.weight * c.demand + dualObjective rest + ≤ c.weight * planSum c.coeff acts + planSum (actionDualLoad rest) acts := + Nat.add_le_add hwc hrest + _ = planSum (actionDualLoad (c :: rest)) acts := hsum + +theorem Mcproof.Spec.operatorCounting_dual_scaled_sound + {A : Type} (cost : A → Nat) (scale : Nat) (cs : List (DualConstraint A)) (acts : List A) + (hlandmarks : ∀ c ∈ cs, c.demand ≤ planSum c.coeff acts) + (hfeasible : ∀ a, actionDualLoad cs a ≤ scale * cost a) : + dualObjective cs ≤ scale * planSum cost acts := by + have h1 : dualObjective cs ≤ planSum (actionDualLoad cs) acts := + dualObjective_le_actionDualLoad_sum cs acts hlandmarks + have h2 : planSum (actionDualLoad cs) acts ≤ planSum (fun a => scale * cost a) acts := + planSum_mono acts hfeasible + have h3 : planSum (fun a => scale * cost a) acts = scale * planSum cost acts := + (planSum_mul_left scale cost acts).symm + omega + +theorem Mcproof.Spec.operatorCounting_dual_div_admissible + {A : Type} (cost : A → Nat) (scale : Nat) (cs : List (DualConstraint A)) (acts : List A) + (hscale : 0 < scale) + (hlandmarks : ∀ c ∈ cs, c.demand ≤ planSum c.coeff acts) + (hfeasible : ∀ a, actionDualLoad cs a ≤ scale * cost a) : + dualObjective cs / scale ≤ planSum cost acts := by + have hscaled : dualObjective cs ≤ scale * planSum cost acts := + Mcproof.Spec.operatorCounting_dual_scaled_sound cost scale cs acts hlandmarks hfeasible + have hdiv : dualObjective cs / scale * scale ≤ dualObjective cs := + Nat.div_mul_le_self (dualObjective cs) scale + have hprod : dualObjective cs / scale * scale ≤ planSum cost acts * scale := by + have hcomm : scale * planSum cost acts = planSum cost acts * scale := Nat.mul_comm scale (planSum cost acts) + omega + exact Nat.le_of_mul_le_mul_right hprod hscale + +theorem Mcproof.Spec.dualFloorNat_sound_and_witness + (scale planCost : Nat) (pairs : List (Nat × Nat)) + (hscale : 0 < scale) + (hbound : dualObjectivePairs pairs ≤ scale * planCost) : + dualFloorNat scale pairs ≤ planCost ∧ dualFloorNat 2 [(2, 9), (1, 8)] = 13 := by + refine ⟨?_, ?_⟩ + · unfold dualFloorNat + have hne : (scale == 0) = false := by + cases scale with + | zero => omega + | succ _ => rfl + rw [hne] + simp only [Bool.false_eq_true, ↓reduceIte] + have hdiv : dualObjectivePairs pairs / scale * scale ≤ dualObjectivePairs pairs := + Nat.div_mul_le_self (dualObjectivePairs pairs) scale + have hprod : dualObjectivePairs pairs / scale * scale ≤ planCost * scale := by + have hcomm : scale * planCost = planCost * scale := Nat.mul_comm scale planCost + omega + exact Nat.le_of_mul_le_mul_right hprod hscale + · rfl + +private theorem cyclicStep_val {k u v : Nat} (hu : u < k) (hv : v < k) : + (v + k - u) % k = if u ≤ v then v - u else v + k - u := by + by_cases huv : u ≤ v + · simp only [huv, ↓reduceIte] + have h1 : v + k - u = (v - u) + k := by omega + rw [h1, Nat.add_mod_right] + exact Nat.mod_eq_of_lt (by omega) + · simp only [huv, ↓reduceIte] + exact Nat.mod_eq_of_lt (by omega) + +theorem Mcproof.Spec.cyclicSwitchTax_triangle + (k a b c : Nat) (_hk : 0 < k) (ha : a < k) (hb : b < k) (hc : c < k) : + cyclicSwitchTax k a c ≤ cyclicSwitchTax k a b + cyclicSwitchTax k b c := by + unfold cyclicSwitchTax + rw [cyclicStep_val ha hc, cyclicStep_val ha hb, cyclicStep_val hb hc] + by_cases hab : a ≤ b <;> by_cases hbc : b ≤ c <;> by_cases hac : a ≤ c <;> + simp only [hab, hbc, hac, ↓reduceIte] <;> omega + +theorem Mcproof.Spec.cyclicSwitchDist_metric_and_witness + (k a b c : Nat) (hk : 0 < k) : + cyclicSwitchDist k a a = 0 ∧ + cyclicSwitchDist k a c ≤ cyclicSwitchDist k a b + cyclicSwitchDist k b c ∧ + cyclicSwitchDist 6 1 5 = 4 ∧ + cyclicSwitchDist 6 5 1 = 2 := by + have hne : (k == 0) = false := by + cases k with + | zero => omega + | succ _ => rfl + refine ⟨?_, ?_, rfl, rfl⟩ + · unfold cyclicSwitchDist cyclicSwitchTax + rw [hne] + simp only [Bool.false_eq_true, ↓reduceIte] + have hma : a % k < k := Nat.mod_lt a hk + rw [cyclicStep_val hma hma] + simp + · unfold cyclicSwitchDist + rw [hne] + simp only [Bool.false_eq_true, ↓reduceIte] + exact Mcproof.Spec.cyclicSwitchTax_triangle k (a % k) (b % k) (c % k) hk + (Nat.mod_lt a hk) (Nat.mod_lt b hk) (Nat.mod_lt c hk) + +private theorem countTrue_mono {X : Type} (p q : X → Bool) (xs : List X) + (himp : ∀ x ∈ xs, p x = true → q x = true) : + countTrue p xs ≤ countTrue q xs := by + induction xs with + | nil => simp [countTrue] + | cons x rest ih => + have hx : (if p x = true then 1 else 0) ≤ (if q x = true then 1 else 0) := by + cases hp : p x with + | false => simp + | true => + have hqx : q x = true := himp x (by simp) hp + simp [hqx] + have hrest : countTrue p rest ≤ countTrue q rest := by + apply ih + intro y hy + exact himp y (by simp [hy]) + simp only [countTrue] + omega + +private theorem stepConsistent_implies_admissible + {S A : Type} (step : S → A → Option S) (isGoal : S → Prop) (f : S → Nat) + (hstep : ∀ s a s', step s a = some s' → f s ≤ f s' + 1) + (hgoal : ∀ g, isGoal g → f g = 0) : + ∀ s acts g, ValidPath step s acts g → isGoal g → f s ≤ acts.length := by + intro s acts g hpath + induction hpath with + | nil s0 => + intro hg + have hz : f s0 = 0 := hgoal s0 hg + simp [hz] + | @cons s0 s1 g0 a as hs _ ih => + intro hg + have h1 : f s0 ≤ f s1 + 1 := hstep s0 a s1 hs + have h2 : f s1 ≤ as.length := ih hg + simp only [List.length_cons] + omega + +theorem Mcproof.Spec.conservation_harvest_potential_repair + {S A X : Type} (step : S → A → Option S) (isGoal : S → Prop) + (U I : List X) (inA inB intact : S → X → Bool) (totalAtoms : Nat) + (h : S → Nat) (c : Nat) + (hconserve : ∀ s, countTrue (inA s) U + countTrue (inB s) U = totalAtoms) + (hirrev : ∀ s a s', step s a = some s' → ∀ x ∈ I, intact s' x = true → intact s x = true) + (hdrop : ∀ s a s', step s a = some s' → + h s ≤ h s' + 1 + c * (intactSupply I intact s - intactSupply I intact s')) + (hgoal : ∀ g, isGoal g → h g = 0) : + (∀ s, countTrue (inA s) U = totalAtoms - countTrue (inB s) U) ∧ + (∀ s a s', step s a = some s' → intactSupply I intact s' ≤ intactSupply I intact s) ∧ + (∀ s a s', step s a = some s' → + prefundedHeuristic h c (intactSupply I intact) s ≤ + prefundedHeuristic h c (intactSupply I intact) s' + 1) ∧ + (∀ s acts g, ValidPath step s acts g → isGoal g → + prefundedHeuristic h c (intactSupply I intact) s ≤ acts.length) := by + have hmono_all : ∀ s a s', step s a = some s' → intactSupply I intact s' ≤ intactSupply I intact s := by + intro s a s' hs + unfold intactSupply + exact countTrue_mono (intact s') (intact s) I (hirrev s a s' hs) + have hcons_all : ∀ s a s', step s a = some s' → + prefundedHeuristic h c (intactSupply I intact) s ≤ + prefundedHeuristic h c (intactSupply I intact) s' + 1 := by + intro s a s' hs + exact Mcproof.Spec.prefundedHeuristic_step_consistent h c (intactSupply I intact) s s' + (hmono_all s a s' hs) (hdrop s a s' hs) + refine ⟨?_, hmono_all, hcons_all, ?_⟩ + · intro s + have hc_eq : countTrue (inA s) U + countTrue (inB s) U = totalAtoms := hconserve s + omega + · apply stepConsistent_implies_admissible step isGoal (prefundedHeuristic h c (intactSupply I intact)) hcons_all + intro g hg + have hz : h g = 0 := hgoal g hg + unfold prefundedHeuristic prefundedVal + omega + +private theorem cut_telescope_bound + {V : Type} (vertices : List V) (occ S_cut : Nat → V → Bool) (cutCap : Nat → Nat) : + ∀ T : Nat, + (∀ t, t < T → + tokensInCut vertices (occ t) (S_cut t) ≤ + tokensInCut vertices (occ (t + 1)) (S_cut (t + 1)) + cutCap t) → + tokensInCut vertices (occ 0) (S_cut 0) ≤ + tokensInCut vertices (occ T) (S_cut T) + horizonCutCapacity cutCap T := by + intro T + induction T with + | zero => + intro _ + simp [horizonCutCapacity] + | succ n ih => + intro hstep + have hprev : + tokensInCut vertices (occ 0) (S_cut 0) ≤ + tokensInCut vertices (occ n) (S_cut n) + horizonCutCapacity cutCap n := + ih (fun t ht => hstep t (by omega)) + have hlast : + tokensInCut vertices (occ n) (S_cut n) ≤ + tokensInCut vertices (occ (n + 1)) (S_cut (n + 1)) + cutCap n := + hstep n (by omega) + simp only [horizonCutCapacity] + omega + +private theorem horizonCutCapacity_le_mul (cutCap : Nat → Nat) (capPerStep : Nat) : + ∀ T : Nat, (∀ t, t < T → cutCap t ≤ capPerStep) → horizonCutCapacity cutCap T ≤ capPerStep * T := by + intro T + induction T with + | zero => + intro _ + simp [horizonCutCapacity] + | succ n ih => + intro hcap + have hprev : horizonCutCapacity cutCap n ≤ capPerStep * n := + ih (fun t ht => hcap t (by omega)) + have hlast : cutCap n ≤ capPerStep := hcap n (by omega) + simp only [horizonCutCapacity, Nat.mul_succ] + omega + +theorem Mcproof.Spec.vertexSplit_unit_flow_horizon_bound + {V : Type} (vertices : List V) (occ blocked S_cut : Nat → V → Bool) + (cutCap : Nat → Nat) (capPerStep demand T : Nat) + (hunblocked : ∀ t v, blocked t v = true → occ t v = false) + (hstep_cut : ∀ t, t < T → + tokensInCut vertices (occ t) (S_cut t) ≤ + tokensInCut vertices (occ (t + 1)) (S_cut (t + 1)) + cutCap t) + (hcap_bound : ∀ t, t < T → cutCap t ≤ capPerStep) + (hinit : demand ≤ tokensInCut vertices (occ 0) (S_cut 0)) + (hsink : tokensInCut vertices (occ T) (S_cut T) = 0) : + (∀ t v, vertexArcFlow occ t v ≤ 1) ∧ + (∀ t v, blocked t v = true → vertexArcFlow occ t v = 0) ∧ + demand ≤ horizonCutCapacity cutCap T ∧ + demand ≤ capPerStep * T := by + have htel : + tokensInCut vertices (occ 0) (S_cut 0) ≤ + tokensInCut vertices (occ T) (S_cut T) + horizonCutCapacity cutCap T := + cut_telescope_bound vertices occ S_cut cutCap T hstep_cut + have hmul : horizonCutCapacity cutCap T ≤ capPerStep * T := + horizonCutCapacity_le_mul cutCap capPerStep T hcap_bound + refine ⟨?_, ?_, ?_, ?_⟩ + · intro t v + unfold vertexArcFlow + cases occ t v <;> simp + · intro t v hb + unfold vertexArcFlow + have hocc : occ t v = false := hunblocked t v hb + simp [hocc] + · omega + · omega + +theorem Mcproof.Spec.zeroSelfLoop_deficit_composition + {S A Q : Type} (step : S → A → Option S) (isGoal : S → Prop) (isExt : A → Bool) + (pi : S → Q) (hQ : Q → Nat) (K : Nat) (m : S → Nat) + (hext_loop : ∀ s a s', step s a = some s' → isExt a = true → pi s' = pi s ∧ m s' ≤ m s + 1) + (hint_step : ∀ s a s', step s a = some s' → isExt a = false → hQ (pi s) ≤ hQ (pi s') + 1 ∧ m s' ≤ m s) + (hgoal : ∀ g, isGoal g → hQ (pi g) = 0 ∧ K ≤ m g) : + (∀ g, isGoal g → composedDeficitHeuristic pi hQ K m g = 0) ∧ + (∀ s a s', step s a = some s' → + composedDeficitHeuristic pi hQ K m s ≤ composedDeficitHeuristic pi hQ K m s' + 1) ∧ + (∀ s acts g, ValidPath step s acts g → isGoal g → + composedDeficitHeuristic pi hQ K m s ≤ acts.length) := by + have hgoal_z : ∀ g, isGoal g → composedDeficitHeuristic pi hQ K m g = 0 := by + intro g hg + obtain ⟨hq0, hmk⟩ := hgoal g hg + unfold composedDeficitHeuristic quotientHeuristic deficitNat + omega + have hstep_c : ∀ s a s', step s a = some s' → + composedDeficitHeuristic pi hQ K m s ≤ composedDeficitHeuristic pi hQ K m s' + 1 := by + intro s a s' hs + unfold composedDeficitHeuristic quotientHeuristic deficitNat + cases he : isExt a with + | true => + obtain ⟨hpi, hm⟩ := hext_loop s a s' hs he + rw [hpi] + omega + | false => + obtain ⟨hq, hm⟩ := hint_step s a s' hs he + omega + refine ⟨hgoal_z, hstep_c, ?_⟩ + exact stepConsistent_implies_admissible step isGoal (composedDeficitHeuristic pi hQ K m) hstep_c hgoal_z + +theorem Mcproof.Spec.ceilDiv_step_consistent_and_admissible + {S : Type} (num : S → Nat) (scale : Nat) (s s' g : S) (planLen : Nat) + (hscale : 0 < scale) + (hgoal : num g = 0) + (hstep : num s ≤ num s' + scale) + (hbound : num s ≤ scale * planLen) : + ceilDivNat (num g) scale = 0 ∧ + ceilDivNat (num s) scale ≤ ceilDivNat (num s') scale + 1 ∧ + ceilDivNat (num s) scale ≤ planLen ∧ + ceilDivNat 13 6 = 3 := by + have hne : (scale == 0) = false := by + cases scale with + | zero => omega + | succ _ => rfl + refine ⟨?_, ?_, ?_, rfl⟩ + · unfold ceilDivNat + rw [hne, hgoal] + simp only [Bool.false_eq_true, ↓reduceIte] + exact Nat.div_eq_of_lt (by omega) + · unfold ceilDivNat + rw [hne] + simp only [Bool.false_eq_true, ↓reduceIte] + have hle : num s + scale - 1 ≤ (num s' + scale - 1) + scale := by omega + have hdiv : (num s + scale - 1) / scale ≤ ((num s' + scale - 1) + scale) / scale := + Nat.div_le_div_right hle + rw [Nat.add_div_right (num s' + scale - 1) hscale] at hdiv + exact hdiv + · unfold ceilDivNat + rw [hne] + simp only [Bool.false_eq_true, ↓reduceIte] + have hmul : (num s + scale - 1) / scale * scale ≤ num s + scale - 1 := + Nat.div_mul_le_self (num s + scale - 1) scale + have hcomm : scale * planLen = planLen * scale := Nat.mul_comm scale planLen + have hlt : (num s + scale - 1) / scale * scale < (planLen + 1) * scale := by + rw [Nat.add_mul, Nat.one_mul] + omega + have hlt_q : (num s + scale - 1) / scale < planLen + 1 := by + by_cases hge : planLen + 1 ≤ (num s + scale - 1) / scale + · have hcontra : (planLen + 1) * scale ≤ (num s + scale - 1) / scale * scale := + Nat.mul_le_mul_right scale hge + omega + · omega + omega + +-- AdapterBounds helper lemmas & proofs (theorems 23..29) ---------------------- + +private theorem vecDot_comm : ∀ (u v : List Int), vecDot u v = vecDot v u + | [], [] => rfl + | [], _ :: _ => rfl + | _ :: _, [] => rfl + | x :: xs, y :: ys => by + simp [vecDot, Int.mul_comm, vecDot_comm xs ys] + +private theorem vecDot_scale_right (c : Int) : ∀ (u v : List Int), + vecDot u (vecScale c v) = c * vecDot u v + | [], _ => by simp [vecDot] + | _ :: _, [] => by simp [vecDot, vecScale] + | _ :: xs, _ :: ys => by + simp [vecDot, vecScale, vecDot_scale_right c xs ys, Int.mul_add, Int.mul_left_comm] + +private theorem vecDot_sub_right : ∀ (h u v : List Int), + u.length = v.length → + vecDot h (vecSub u v) = vecDot h u - vecDot h v + | [], _, _, _ => by simp [vecDot] + | _ :: _, [], [], _ => by simp [vecDot, vecSub] + | _ :: _, [], _ :: _, hlen => by cases hlen + | _ :: _, _ :: _, [], hlen => by cases hlen + | _ :: xs, _ :: as, _ :: bs, hlen => by + simp [vecDot, vecSub, vecDot_sub_right xs as bs (Nat.succ.inj hlen), Int.mul_sub] + omega + +private theorem vecScale_length (c : Int) : ∀ (v : List Int), (vecScale c v).length = v.length + | [] => rfl + | _ :: xs => by simp [vecScale, vecScale_length c xs] + +private theorem vecAdd_length : ∀ (u v : List Int), u.length = v.length → (vecAdd u v).length = u.length + | [], [], _ => rfl + | [], _ :: _, h => by cases h + | _ :: _, [], h => by cases h + | _ :: xs, _ :: ys, h => by + simp [vecAdd, vecAdd_length xs ys (Nat.succ.inj h)] + +private theorem vecHadamard_length : ∀ (g w : List Int), g.length = w.length → (vecHadamard g w).length = g.length + | [], [], _ => rfl + | [], _ :: _, h => by cases h + | _ :: _, [], h => by cases h + | _ :: gs, _ :: ws, h => by simp [vecHadamard, vecHadamard_length gs ws (Nat.succ.inj h)] + +private theorem vecDot_add_right : ∀ (w u v : List Int), + u.length = v.length → + vecDot w (vecAdd u v) = vecDot w u + vecDot w v + | [], _, _, _ => by simp [vecDot] + | _ :: _, [], [], _ => by simp [vecDot, vecAdd] + | _ :: _, [], _ :: _, h => by cases h + | _ :: _, _ :: _, [], h => by cases h + | _ :: zs, _ :: xs, _ :: ys, h => by + simp [vecDot, vecAdd, vecDot_add_right zs xs ys (Nat.succ.inj h), Int.mul_add] + omega + +private theorem vecDot_add_left : ∀ (u v w : List Int), + u.length = v.length → + vecDot (vecAdd u v) w = vecDot u w + vecDot v w + | [], [], _, _ => by simp [vecDot, vecAdd] + | [], _ :: _, _, h => by cases h + | _ :: _, [], _, h => by cases h + | _ :: _, _ :: _, [], _ => by simp [vecDot, vecAdd] + | _ :: xs, _ :: ys, _ :: zs, h => by + simp [vecDot, vecAdd, vecDot_add_left xs ys zs (Nat.succ.inj h), Int.add_mul] + omega + +private theorem vecNormSq_add : ∀ (u v : List Int), + u.length = v.length → + vecNormSq (vecAdd u v) = vecNormSq u + 2 * vecDot u v + vecNormSq v := by + intro u v hlen + unfold vecNormSq + rw [vecDot_add_left u v (vecAdd u v) hlen] + rw [vecDot_add_right u u v hlen, vecDot_add_right v u v hlen] + rw [vecDot_comm v u] + omega + +private theorem tangent_orth_aux (h delta : List Int) (hlen : h.length = delta.length) : + vecDot h (tangentProjScaled h delta) = 0 := by + unfold tangentProjScaled + have hlen_sc : (vecScale (vecNormSq h) delta).length = (vecScale (vecDot h delta) h).length := by + rw [vecScale_length, vecScale_length, hlen] + rw [vecDot_sub_right h _ _ hlen_sc, vecDot_scale_right, vecDot_scale_right] + unfold vecNormSq + rw [Int.mul_comm (vecDot h h) (vecDot h delta)] + omega + +theorem Mcproof.Spec.rmsnorm_tangent_orthogonal + (h delta : List Int) (hlen : h.length = delta.length) : + vecDot h (tangentProjScaled h delta) = 0 ∧ + tangentProjScaled [3, 4] [5, 0] = [80, -60] := by + exact ⟨tangent_orth_aux h delta hlen, by decide⟩ + +theorem Mcproof.Spec.rmsnorm_orthogonal_stack_norm_inflation + (h d1 d2 d3 : List Int) + (hl1 : h.length = d1.length) (hl2 : h.length = d2.length) (hl3 : h.length = d3.length) + (ho1 : vecDot h d1 = 0) (ho2 : vecDot h d2 = 0) (ho3 : vecDot h d3 = 0) + (h12 : vecDot d1 d2 = 0) (h13 : vecDot d1 d3 = 0) (h23 : vecDot d2 d3 = 0) : + vecNormSq (vecAdd h d1) = vecNormSq h + vecNormSq d1 ∧ + vecNormSq (vecAdd (vecAdd (vecAdd h d1) d2) d3) = + vecNormSq h + vecNormSq d1 + vecNormSq d2 + vecNormSq d3 := by + have h_hd1 : vecNormSq (vecAdd h d1) = vecNormSq h + vecNormSq d1 := by + rw [vecNormSq_add h d1 hl1, ho1] + omega + refine ⟨h_hd1, ?_⟩ + have hlen_hd1 : (vecAdd h d1).length = d2.length := by + rw [vecAdd_length h d1 hl1, hl2] + have hdot_hd1_d2 : vecDot (vecAdd h d1) d2 = 0 := by + rw [vecDot_add_left h d1 d2 hl1, ho2, h12] + omega + have h_hd12 : vecNormSq (vecAdd (vecAdd h d1) d2) = vecNormSq h + vecNormSq d1 + vecNormSq d2 := by + rw [vecNormSq_add (vecAdd h d1) d2 hlen_hd1, hdot_hd1_d2, h_hd1] + omega + have hlen_hd12 : (vecAdd (vecAdd h d1) d2).length = d3.length := by + rw [vecAdd_length (vecAdd h d1) d2 hlen_hd1, vecAdd_length h d1 hl1, hl3] + have hdot_hd12_d3 : vecDot (vecAdd (vecAdd h d1) d2) d3 = 0 := by + rw [vecDot_add_left (vecAdd h d1) d2 d3 hlen_hd1, vecDot_add_left h d1 d3 hl1, ho3, h13, h23] + omega + rw [vecNormSq_add (vecAdd (vecAdd h d1) d2) d3 hlen_hd12, hdot_hd12_d3, h_hd12] + omega + +private theorem sumNat_replicate : ∀ (L d : Nat), sumNat (List.replicate L d) = L * d + | 0, _ => by simp [sumNat] + | L + 1, d => by + simp [List.replicate_succ, sumNat, sumNat_replicate L d, Nat.add_mul] + omega + +private theorem sumSqNat_replicate : ∀ (L d : Nat), sumSqNat (List.replicate L d) = L * (d * d) + | 0, _ => by simp [sumSqNat] + | L + 1, d => by + simp [List.replicate_succ, sumSqNat, sumSqNat_replicate L d, Nat.add_mul] + omega + +private theorem sumSqNat_le_sq_sumNat : ∀ (ds : List Nat), sumSqNat ds ≤ sumNat ds * sumNat ds + | [] => by simp [sumSqNat, sumNat] + | x :: xs => by + have ih := sumSqNat_le_sq_sumNat xs + simp [sumSqNat, sumNat] + have h_expand : (x + sumNat xs) * (x + sumNat xs) = + x * x + sumNat xs * sumNat xs + 2 * (x * sumNat xs) := by + rw [Nat.add_mul, Nat.mul_add, Nat.mul_add, Nat.mul_comm (sumNat xs) x] + omega + omega + +theorem Mcproof.Spec.multilayer_quadratic_shrinkage_reduction (L d : Nat) (ds : List Nat) : + L * sumSqNat (List.replicate L d) = sumNat (List.replicate L d) * sumNat (List.replicate L d) ∧ + sumSqNat ds ≤ sumNat ds * sumNat ds ∧ + (0 < d → 2 ≤ L → sumSqNat (List.replicate L d) < sumNat (List.replicate L d) * sumNat (List.replicate L d)) := by + refine ⟨?_, sumSqNat_le_sq_sumNat ds, ?_⟩ + · rw [sumSqNat_replicate, sumNat_replicate] + simp [Nat.mul_comm, Nat.mul_left_comm] + · intro hd hL + rw [sumSqNat_replicate, sumNat_replicate] + have hdd : 0 < d * d := Nat.mul_pos hd hd + have hpos : 0 < L * (d * d) := Nat.mul_pos (by omega) hdd + have h1 : 1 * (L * (d * d)) < L * (L * (d * d)) := + Nat.mul_lt_mul_of_pos_right (by omega : 1 < L) hpos + have h_assoc : L * (L * (d * d)) = (L * d) * (L * d) := by + simp [Nat.mul_comm, Nat.mul_left_comm] + omega + +private theorem hadamard_dot_eq_sqDiag_metric : ∀ (g u v : List Int), + vecDot (vecHadamard g u) (vecHadamard g v) = vecDot u (vecHadamard (vecSqDiag g) v) + | [], [], _ => rfl + | [], _ :: _, _ => rfl + | _ :: _, [], _ => rfl + | _ :: _, _ :: _, [] => by simp [vecDot, vecHadamard, vecSqDiag] + | x :: xs, a :: as, b :: bs => by + simp [vecDot, vecHadamard, vecSqDiag, hadamard_dot_eq_sqDiag_metric xs as bs] + have hterm : (x * a) * (x * b) = a * (x * x * b) := by + simp [Int.mul_comm, Int.mul_left_comm] + rw [hterm] + +theorem Mcproof.Spec.swiglu_gate_metric_and_counterexample (g u v : List Int) : + vecDot (vecHadamard g u) (vecHadamard g v) = vecDot u (vecHadamard (vecSqDiag g) v) ∧ + vecDot [1, 1] [1, -1] = 0 ∧ + vecDot (vecHadamard [2, 1] [1, 1]) (vecHadamard [2, 1] [1, -1]) = 3 := by + exact ⟨hadamard_dot_eq_sqDiag_metric g u v, by decide, by decide⟩ + +private theorem vecDot_hadamard_adjoint : ∀ (g u w : List Int), + vecDot (vecHadamard g u) w = vecDot u (vecHadamard g w) + | [], [], _ => rfl + | [], _ :: _, _ => rfl + | _ :: _, [], _ => rfl + | _ :: _, _ :: _, [] => by simp [vecDot, vecHadamard] + | x :: xs, a :: as, b :: bs => by + simp [vecDot, vecHadamard, vecDot_hadamard_adjoint xs as bs] + have h_assoc : x * a * b = a * (x * b) := by + simp [Int.mul_comm, Int.mul_left_comm] + rw [h_assoc] + +theorem Mcproof.Spec.swiglu_radial_null_orthogonality + (g u w : List Int) (hgu : g.length = u.length) (hgw : g.length = w.length) : + vecDot (vecHadamard g (tangentProjScaled (vecHadamard g w) u)) w = 0 := by + rw [vecDot_hadamard_adjoint, vecDot_comm] + apply tangent_orth_aux + rw [vecHadamard_length g w hgw, hgu] + +theorem Mcproof.Spec.scalar_projection_retention_illusion : + vecDot [1, 0] [0, 32] = 0 ∧ + vecDot (vecAdd [1, 0] [0, 32]) [1, 0] = vecNormSq [1, 0] ∧ + vecNormSq (vecSub (vecAdd [1, 0] [0, 32]) [1, 0]) = 1024 * vecNormSq [1, 0] := by + decide + +private theorem sq_nonneg (x : Int) : 0 ≤ x * x := + if hx : 0 ≤ x then + Int.mul_nonneg hx hx + else + have hnx : 0 ≤ -x := by omega + have heq : (-x) * (-x) = x * x := by rw [Int.neg_mul_neg] + heq ▸ Int.mul_nonneg hnx hnx + +private theorem nonneg_of_sq_le_sq {a b : Int} (hb : 0 ≤ b) (h : a * a ≤ b * b) : a ≤ b := + if hlt : b < a then + have hpos1 : 0 < a - b := by omega + have hpos2 : 0 < a + b := by omega + have hmul : 0 < (a - b) * (a + b) := Int.mul_pos hpos1 hpos2 + have h_ab : (a - b) * (a + b) = a * a - b * b := by + rw [Int.sub_mul, Int.mul_add, Int.mul_add, Int.mul_comm b a] + omega + by omega + else by omega + +private theorem poly_lower_identity (R2 D2 : Int) : + (2 * R2 + D2) * (2 * R2 + D2) = 4 * (R2 + D2) * R2 + D2 * D2 := by + have h1 : (2 * R2 + D2) * (2 * R2 + D2) = + 2 * R2 * (2 * R2) + (2 * R2 * D2 + D2 * (2 * R2)) + D2 * D2 := by + rw [Int.add_mul, Int.mul_add, Int.mul_add] + omega + have h2 : 2 * R2 * (2 * R2) = 4 * (R2 * R2) := by + have : (2 : Int) * R2 * (2 * R2) = (2 * 2) * (R2 * R2) := by + simp [Int.mul_assoc, Int.mul_comm, Int.mul_left_comm] + exact this + have h3 : 2 * R2 * D2 + D2 * (2 * R2) = 4 * (R2 * D2) := by + have ha : (2 : Int) * R2 * D2 = 2 * (R2 * D2) := by simp [Int.mul_assoc] + have hb : D2 * ((2 : Int) * R2) = 2 * (R2 * D2) := by + simp [Int.mul_assoc, Int.mul_comm, Int.mul_left_comm] + omega + have h4 : 4 * (R2 + D2) * R2 = 4 * (R2 * R2) + 4 * (R2 * D2) := by + have hc : (4 : Int) * (R2 + D2) * R2 = 4 * ((R2 + D2) * R2) := by simp [Int.mul_assoc] + rw [hc, Int.add_mul, Int.mul_add] + have hd : D2 * R2 = R2 * D2 := Int.mul_comm D2 R2 + rw [hd] + omega + +private theorem poly_upper_identity (R2 D2 : Int) : + (R2 + D2) * ((2 * R2 - D2) * (2 * R2 - D2)) + D2 * D2 * (3 * R2 - D2) = + 4 * R2 * (R2 * R2) := by + have hsq : (2 * R2 - D2) * (2 * R2 - D2) = 4 * (R2 * R2) - 4 * (R2 * D2) + D2 * D2 := by + rw [Int.sub_mul, Int.mul_sub, Int.mul_sub] + have h22 : (2 : Int) * R2 * (2 * R2) = 4 * (R2 * R2) := by + simp [Int.mul_assoc, Int.mul_comm, Int.mul_left_comm] + have h2d : (2 : Int) * R2 * D2 = 2 * (R2 * D2) := by simp [Int.mul_assoc] + have hd2 : D2 * ((2 : Int) * R2) = 2 * (R2 * D2) := by + simp [Int.mul_assoc, Int.mul_comm, Int.mul_left_comm] + omega + rw [hsq, Int.add_mul] + have hr : R2 * (4 * (R2 * R2) - 4 * (R2 * D2) + D2 * D2) = + 4 * R2 * (R2 * R2) - 4 * (R2 * (R2 * D2)) + R2 * (D2 * D2) := by + rw [Int.mul_add, Int.mul_sub] + have hr1 : R2 * ((4 : Int) * (R2 * R2)) = 4 * R2 * (R2 * R2) := by + simp [Int.mul_assoc, Int.mul_comm, Int.mul_left_comm] + have hr2 : R2 * ((4 : Int) * (R2 * D2)) = 4 * (R2 * (R2 * D2)) := by + simp [Int.mul_assoc, Int.mul_comm] + rw [hr1, hr2] + have hd : D2 * (4 * (R2 * R2) - 4 * (R2 * D2) + D2 * D2) = + 4 * (R2 * (R2 * D2)) - 4 * (R2 * (D2 * D2)) + D2 * (D2 * D2) := by + rw [Int.mul_add, Int.mul_sub] + have hd1 : D2 * ((4 : Int) * (R2 * R2)) = 4 * (R2 * (R2 * D2)) := by + simp [Int.mul_assoc, Int.mul_comm, Int.mul_left_comm] + have hd2 : D2 * ((4 : Int) * (R2 * D2)) = 4 * (R2 * (D2 * D2)) := by + simp [Int.mul_assoc, Int.mul_comm, Int.mul_left_comm] + rw [hd1, hd2] + have hrem : D2 * D2 * (3 * R2 - D2) = 3 * (R2 * (D2 * D2)) - D2 * (D2 * D2) := by + rw [Int.mul_sub] + have hm1 : D2 * D2 * ((3 : Int) * R2) = 3 * (R2 * (D2 * D2)) := by + simp [Int.mul_assoc, Int.mul_comm, Int.mul_left_comm] + have hm2 : D2 * D2 * D2 = D2 * (D2 * D2) := by simp [Int.mul_assoc] + rw [hm1, hm2] + omega + +theorem Mcproof.Spec.rmsnorm_rational_shrinkage_sandwich + (R2 D2 qn qd : Int) + (hR : 0 < R2) (hD : 0 ≤ D2) (hqn : 0 < qn) (hqd : 0 < qd) + (heq : (R2 + D2) * (qn * qn) = R2 * (qd * qd)) : + shrinkageSandwichCheck R2 D2 qn qd = true ∧ + D2 * qd ≤ 2 * (R2 + D2) * (qd - qn) ∧ + 2 * R2 * (qd - qn) ≤ D2 * qd := by + have hpair : D2 * qd ≤ 2 * (R2 + D2) * (qd - qn) ∧ 2 * R2 * (qd - qn) ≤ D2 * qd := by + constructor + · have hD2_nn : 0 ≤ (D2 * qd) * (D2 * qd) := sq_nonneg (D2 * qd) + have h_lhs : (2 * (R2 + D2) * qn) * (2 * (R2 + D2) * qn) = + 4 * (R2 + D2) * ((R2 + D2) * (qn * qn)) := by + simp [Int.mul_assoc, Int.mul_comm, Int.mul_left_comm] + have h_rhs : ((2 * R2 + D2) * qd) * ((2 * R2 + D2) * qd) = + ((2 * R2 + D2) * (2 * R2 + D2)) * (qd * qd) := by + simp [Int.mul_assoc, Int.mul_comm, Int.mul_left_comm] + have h_sq : (2 * (R2 + D2) * qn) * (2 * (R2 + D2) * qn) ≤ + ((2 * R2 + D2) * qd) * ((2 * R2 + D2) * qd) := by + rw [h_lhs, heq, h_rhs, poly_lower_identity R2 D2, Int.add_mul] + have ha : 4 * (R2 + D2) * (R2 * (qd * qd)) = 4 * (R2 + D2) * R2 * (qd * qd) := by + simp [Int.mul_assoc] + have hb : D2 * D2 * (qd * qd) = (D2 * qd) * (D2 * qd) := by + simp [Int.mul_comm, Int.mul_left_comm] + omega + have hb_nn : 0 ≤ (2 * R2 + D2) * qd := Int.mul_nonneg (by omega) (by omega) + have h_lin := nonneg_of_sq_le_sq hb_nn h_sq + have h_exp1 : 2 * (R2 + D2) * (qd - qn) = 2 * (R2 + D2) * qd - 2 * (R2 + D2) * qn := + Int.mul_sub (2 * (R2 + D2)) qd qn + have h_exp2 : 2 * (R2 + D2) * qd = (2 * R2 + D2) * qd + D2 * qd := by + have : (2 : Int) * (R2 + D2) = (2 * R2 + D2) + D2 := by omega + rw [this, Int.add_mul] + omega + · exact if hbig : 2 * R2 ≤ D2 then + have hpos : 0 < 2 * R2 * qn := Int.mul_pos (by omega) hqn + have h1 : 2 * R2 * (qd - qn) ≤ 2 * R2 * qd := by + rw [Int.mul_sub] + omega + have h2 : 2 * R2 * qd ≤ D2 * qd := Int.mul_le_mul_of_nonneg_right hbig (by omega) + by omega + else + have h3R : 0 ≤ 3 * R2 - D2 := by omega + have h_rem_nn : 0 ≤ D2 * D2 * (3 * R2 - D2) := Int.mul_nonneg (sq_nonneg D2) h3R + have h_poly_le : (R2 + D2) * ((2 * R2 - D2) * (2 * R2 - D2)) ≤ 4 * R2 * (R2 * R2) := by + have hp := poly_upper_identity R2 D2 + omega + have h_mul_qd := Int.mul_le_mul_of_nonneg_right h_poly_le (sq_nonneg qd) + have h_rhs_sub : 4 * R2 * (R2 * R2) * (qd * qd) = + (R2 + D2) * ((2 * R2 * qn) * (2 * R2 * qn)) := by + have s1 : 4 * R2 * (R2 * R2) * (qd * qd) = (4 * (R2 * R2)) * (R2 * (qd * qd)) := by + simp [Int.mul_assoc, Int.mul_comm, Int.mul_left_comm] + rw [s1, ← heq] + simp [Int.mul_assoc, Int.mul_comm, Int.mul_left_comm] + have h_lhs_sub : (R2 + D2) * ((2 * R2 - D2) * (2 * R2 - D2)) * (qd * qd) = + (R2 + D2) * (((2 * R2 - D2) * qd) * ((2 * R2 - D2) * qd)) := by + simp [Int.mul_assoc, Int.mul_comm, Int.mul_left_comm] + have hRD_pos : 0 < R2 + D2 := by omega + have h_sq2 : ((2 * R2 - D2) * qd) * ((2 * R2 - D2) * qd) ≤ + (2 * R2 * qn) * (2 * R2 * qn) := + if hgt : (2 * R2 * qn) * (2 * R2 * qn) < ((2 * R2 - D2) * qd) * ((2 * R2 - D2) * qd) then + have h_strict := Int.mul_lt_mul_of_pos_left hgt hRD_pos + by rw [h_lhs_sub, h_rhs_sub] at h_mul_qd; omega + else by omega + have hb2_nn : 0 ≤ 2 * R2 * qn := Int.mul_nonneg (by omega) (by omega) + have h_lin2 := nonneg_of_sq_le_sq hb2_nn h_sq2 + have h_rew : (2 * R2 - D2) * qd = 2 * R2 * qd - D2 * qd := Int.sub_mul (2 * R2) D2 qd + have h_rew2 : 2 * R2 * (qd - qn) = 2 * R2 * qd - 2 * R2 * qn := Int.mul_sub (2 * R2) qd qn + by omega + exact ⟨by unfold shrinkageSandwichCheck; exact decide_eq_true hpair, hpair⟩ diff --git a/lean/Mcproof/Spec.lean b/lean/Mcproof/Spec.lean index dee23b6..4db4635 100644 --- a/lean/Mcproof/Spec.lean +++ b/lean/Mcproof/Spec.lean @@ -1,12 +1,14 @@ /- - Challenge specification for the mcproof Lean 4 reference package. - Each safety theorem is paired with a usefulness theorem so a trivial - constant or rejecting implementation fails `lake comparator --paranoid`. + Challenge specification for the mcproof Lean 4 reference package (v0.2). + Each safety or soundness theorem is paired with a usefulness or non-vacuity + theorem so a trivial constant or rejecting implementation fails `lake comparator --paranoid`. -/ import Mcproof open Mcproof +open Mcproof.TransitionBounds +open Mcproof.AdapterBounds /-- Safety: when `lo <= hi`, `clampNat lo hi x` lies inside `[lo, hi]`. -/ theorem Mcproof.Spec.clampNat_in_range (lo hi x : Nat) (h : lo ≤ hi) : @@ -41,3 +43,183 @@ theorem Mcproof.Spec.cleanSegments_safe (segs out : List String) : theorem Mcproof.Spec.cleanSegments_singleton (s : String) (hs : s ≠ "" ∧ s ≠ "." ∧ s ≠ ".." ∧ ¬ s.contains '/' ∧ ¬ s.contains '\x00') : cleanSegments [s] = some [s] := sorry + +/-- Quotient homomorphism global 1-step consistency on the full state space. -/ +theorem Mcproof.Spec.quotientHeuristic_step_consistent + {S A Q : Type} (step : S → A → Option S) (pi : S → Q) (qEdge : Q → Q → Prop) (hQ : Q → Nat) + (hproj : ∀ s a s', step s a = some s' → pi s = pi s' ∨ qEdge (pi s) (pi s')) + (hq : ∀ u v, qEdge u v → hQ u ≤ hQ v + 1) : + ∀ s a s', step s a = some s' → quotientHeuristic pi hQ s ≤ quotientHeuristic pi hQ s' + 1 := sorry + +/-- Quotient homomorphism admissibility across every valid path reaching a goal state. -/ +theorem Mcproof.Spec.quotientHeuristic_admissible + {S A Q : Type} (step : S → A → Option S) (isGoal : S → Prop) + (pi : S → Q) (qEdge : Q → Q → Prop) (hQ : Q → Nat) + (hproj : ∀ s a s', step s a = some s' → pi s = pi s' ∨ qEdge (pi s) (pi s')) + (hq : ∀ u v, qEdge u v → hQ u ≤ hQ v + 1) + (hgoal : ∀ g, isGoal g → hQ (pi g) = 0) : + ∀ s acts g, ValidPath step s acts g → isGoal g → quotientHeuristic pi hQ s ≤ acts.length := sorry + +/-- Pre-funded resource potential goal preservation and admissibility inheritance. -/ +theorem Mcproof.Spec.prefundedHeuristic_admissible_and_goal + {S : Type} (h : S → Nat) (c : Nat) (r : S → Nat) : + (∀ s, prefundedHeuristic h c r s ≤ h s) ∧ + (∀ g, h g = 0 → prefundedHeuristic h c r g = 0) ∧ + (∀ s len, h s ≤ len → prefundedHeuristic h c r s ≤ len) := sorry + +/-- Pre-Funded Monotone Potential Repair Theorem: restores universal 1-step consistency in `Nat`. -/ +theorem Mcproof.Spec.prefundedHeuristic_step_consistent + {S : Type} (h : S → Nat) (c : Nat) (r : S → Nat) (s s' : S) + (hmono : r s' ≤ r s) + (hdrop : h s ≤ h s' + 1 + c * (r s - r s')) : + prefundedHeuristic h c r s ≤ prefundedHeuristic h c r s' + 1 := sorry + +/-- Non-vacuity witness for bounded 2-step drop repair (`h: 12 -> 10` repaired to `11 -> 10` under `r: 1 -> 0`). -/ +theorem Mcproof.Spec.prefundedVal_bounded_drop_witness : + prefundedVal 12 1 1 = 11 ∧ + prefundedVal 10 1 0 = 10 ∧ + prefundedVal 12 1 1 ≤ prefundedVal 10 1 0 + 1 := sorry + +/-- Non-negative operator-counting / Farkas LP dual scaled lower-bound soundness. -/ +theorem Mcproof.Spec.operatorCounting_dual_scaled_sound + {A : Type} (cost : A → Nat) (scale : Nat) (cs : List (DualConstraint A)) (acts : List A) + (hlandmarks : ∀ c ∈ cs, c.demand ≤ planSum c.coeff acts) + (hfeasible : ∀ a, actionDualLoad cs a ≤ scale * cost a) : + dualObjective cs ≤ scale * planSum cost acts := sorry + +/-- Division-free integer dual floor admissibility `(dualObjective cs) / scale <= planSum cost acts`. -/ +theorem Mcproof.Spec.operatorCounting_dual_div_admissible + {A : Type} (cost : A → Nat) (scale : Nat) (cs : List (DualConstraint A)) (acts : List A) + (hscale : 0 < scale) + (hlandmarks : ∀ c ∈ cs, c.demand ≤ planSum c.coeff acts) + (hfeasible : ∀ a, actionDualLoad cs a ≤ scale * cost a) : + dualObjective cs / scale ≤ planSum cost acts := sorry + +/-- Executable `dualFloorNat` soundness and non-trivial multi-constraint witness (`13`). -/ +theorem Mcproof.Spec.dualFloorNat_sound_and_witness + (scale planCost : Nat) (pairs : List (Nat × Nat)) + (hscale : 0 < scale) + (hbound : dualObjectivePairs pairs ≤ scale * planCost) : + dualFloorNat scale pairs ≤ planCost ∧ dualFloorNat 2 [(2, 9), (1, 8)] = 13 := sorry + +/-- Cyclic mode-switch triangle inequality `cyclicSwitchTax k a c <= cyclicSwitchTax k a b + cyclicSwitchTax k b c` for modes `< k`. -/ +theorem Mcproof.Spec.cyclicSwitchTax_triangle + (k a b c : Nat) (hk : 0 < k) (ha : a < k) (hb : b < k) (hc : c < k) : + cyclicSwitchTax k a c ≤ cyclicSwitchTax k a b + cyclicSwitchTax k b c := sorry + +/-- Unrestricted `cyclicSwitchDist` metric admissibility (zero self-tax, triangle inequality, and `k = 6` witness). -/ +theorem Mcproof.Spec.cyclicSwitchDist_metric_and_witness + (k a b c : Nat) (hk : 0 < k) : + cyclicSwitchDist k a a = 0 ∧ + cyclicSwitchDist k a c ≤ cyclicSwitchDist k a b + cyclicSwitchDist k b c ∧ + cyclicSwitchDist 6 1 5 = 4 ∧ + cyclicSwitchDist 6 5 1 = 2 := sorry + +/-- Two-species conservation identity and global pre-funded potential repair under irreversible harvests on `I`. -/ +theorem Mcproof.Spec.conservation_harvest_potential_repair + {S A X : Type} (step : S → A → Option S) (isGoal : S → Prop) + (U I : List X) (inA inB intact : S → X → Bool) (totalAtoms : Nat) + (h : S → Nat) (c : Nat) + (hconserve : ∀ s, countTrue (inA s) U + countTrue (inB s) U = totalAtoms) + (hirrev : ∀ s a s', step s a = some s' → ∀ x ∈ I, intact s' x = true → intact s x = true) + (hdrop : ∀ s a s', step s a = some s' → + h s ≤ h s' + 1 + c * (intactSupply I intact s - intactSupply I intact s')) + (hgoal : ∀ g, isGoal g → h g = 0) : + (∀ s, countTrue (inA s) U = totalAtoms - countTrue (inB s) U) ∧ + (∀ s a s', step s a = some s' → intactSupply I intact s' ≤ intactSupply I intact s) ∧ + (∀ s a s', step s a = some s' → + prefundedHeuristic h c (intactSupply I intact) s ≤ + prefundedHeuristic h c (intactSupply I intact) s' + 1) ∧ + (∀ s acts g, ValidPath step s acts g → isGoal g → + prefundedHeuristic h c (intactSupply I intact) s ≤ acts.length) := sorry + +/-- Vertex-split unit-capacity flow feasibility and time-expanded min-cut horizon lower bound. -/ +theorem Mcproof.Spec.vertexSplit_unit_flow_horizon_bound + {V : Type} (vertices : List V) (occ blocked S_cut : Nat → V → Bool) + (cutCap : Nat → Nat) (capPerStep demand T : Nat) + (hunblocked : ∀ t v, blocked t v = true → occ t v = false) + (hstep_cut : ∀ t, t < T → + tokensInCut vertices (occ t) (S_cut t) ≤ + tokensInCut vertices (occ (t + 1)) (S_cut (t + 1)) + cutCap t) + (hcap_bound : ∀ t, t < T → cutCap t ≤ capPerStep) + (hinit : demand ≤ tokensInCut vertices (occ 0) (S_cut 0)) + (hsink : tokensInCut vertices (occ T) (S_cut T) = 0) : + (∀ t v, vertexArcFlow occ t v ≤ 1) ∧ + (∀ t v, blocked t v = true → vertexArcFlow occ t v = 0) ∧ + demand ≤ horizonCutCapacity cutCap T ∧ + demand ≤ capPerStep * T := sorry + +/-- Zero-self-loop quotient + monotone deficit composition: preserves goal `0`, 1-step consistency, and admissibility. -/ +theorem Mcproof.Spec.zeroSelfLoop_deficit_composition + {S A Q : Type} (step : S → A → Option S) (isGoal : S → Prop) (isExt : A → Bool) + (pi : S → Q) (hQ : Q → Nat) (K : Nat) (m : S → Nat) + (hext_loop : ∀ s a s', step s a = some s' → isExt a = true → pi s' = pi s ∧ m s' ≤ m s + 1) + (hint_step : ∀ s a s', step s a = some s' → isExt a = false → hQ (pi s) ≤ hQ (pi s') + 1 ∧ m s' ≤ m s) + (hgoal : ∀ g, isGoal g → hQ (pi g) = 0 ∧ K ≤ m g) : + (∀ g, isGoal g → composedDeficitHeuristic pi hQ K m g = 0) ∧ + (∀ s a s', step s a = some s' → + composedDeficitHeuristic pi hQ K m s ≤ composedDeficitHeuristic pi hQ K m s' + 1) ∧ + (∀ s acts g, ValidPath step s acts g → isGoal g → + composedDeficitHeuristic pi hQ K m s ≤ acts.length) := sorry + +/-- Rational-ceiling 1-step consistency and admissibility preservation (`ceil(num(s) / scale)`). -/ +theorem Mcproof.Spec.ceilDiv_step_consistent_and_admissible + {S : Type} (num : S → Nat) (scale : Nat) (s s' g : S) (planLen : Nat) + (hscale : 0 < scale) + (hgoal : num g = 0) + (hstep : num s ≤ num s' + scale) + (hbound : num s ≤ scale * planLen) : + ceilDivNat (num g) scale = 0 ∧ + ceilDivNat (num s) scale ≤ ceilDivNat (num s') scale + 1 ∧ + ceilDivNat (num s) scale ≤ planLen ∧ + ceilDivNat 13 6 = 3 := sorry + +/-- Division-free RMSNorm tangent projection orthogonality ` * h> = 0`. -/ +theorem Mcproof.Spec.rmsnorm_tangent_orthogonal + (h delta : List Int) (hlen : h.length = delta.length) : + vecDot h (tangentProjScaled h delta) = 0 ∧ + tangentProjScaled [3, 4] [5, 0] = [80, -60] := sorry + +/-- Additive squared-norm inflation under pairwise-orthogonal RMSNorm tangent adapter stacks. -/ +theorem Mcproof.Spec.rmsnorm_orthogonal_stack_norm_inflation + (h d1 d2 d3 : List Int) + (hl1 : h.length = d1.length) (hl2 : h.length = d2.length) (hl3 : h.length = d3.length) + (ho1 : vecDot h d1 = 0) (ho2 : vecDot h d2 = 0) (ho3 : vecDot h d3 = 0) + (h12 : vecDot d1 d2 = 0) (h13 : vecDot d1 d3 = 0) (h23 : vecDot d2 d3 = 0) : + vecNormSq (vecAdd h d1) = vecNormSq h + vecNormSq d1 ∧ + vecNormSq (vecAdd (vecAdd (vecAdd h d1) d2) d3) = + vecNormSq h + vecNormSq d1 + vecNormSq d2 + vecNormSq d3 := sorry + +/-- Multi-layer `1/L` quadratic radial-shrinkage reduction over `L` balanced layers. -/ +theorem Mcproof.Spec.multilayer_quadratic_shrinkage_reduction + (L d : Nat) (ds : List Nat) : + L * sumSqNat (List.replicate L d) = sumNat (List.replicate L d) * sumNat (List.replicate L d) ∧ + sumSqNat ds ≤ sumNat ds * sumNat ds ∧ + (0 < d → 2 ≤ L → sumSqNat (List.replicate L d) < sumNat (List.replicate L d) * sumNat (List.replicate L d)) := sorry + +/-- SwiGLU elementwise gate metric ` = ` and pre-gate orthogonality counterexample. -/ +theorem Mcproof.Spec.swiglu_gate_metric_and_counterexample + (g u v : List Int) : + vecDot (vecHadamard g u) (vecHadamard g v) = vecDot u (vecHadamard (vecSqDiag g) v) ∧ + vecDot [1, 1] [1, -1] = 0 ∧ + vecDot (vecHadamard [2, 1] [1, 1]) (vecHadamard [2, 1] [1, -1]) = 3 := sorry + +/-- SwiGLU Hadamard adjoint radial-null hyperplane orthogonality ` = 0`. -/ +theorem Mcproof.Spec.swiglu_radial_null_orthogonality + (g u w : List Int) (hgu : g.length = u.length) (hgw : g.length = w.length) : + vecDot (vecHadamard g (tangentProjScaled (vecHadamard g w) u)) w = 0 := sorry + +/-- Scalar-projection subspace-retention illusion (`100%` inner-product retention with `1024x` orthogonal drift energy). -/ +theorem Mcproof.Spec.scalar_projection_retention_illusion : + vecDot [1, 0] [0, 32] = 0 ∧ + vecDot (vecAdd [1, 0] [0, 32]) [1, 0] = vecNormSq [1, 0] ∧ + vecNormSq (vecSub (vecAdd [1, 0] [0, 32]) [1, 0]) = 1024 * vecNormSq [1, 0] := sorry + +/-- Exact rational sandwich `D2 / (2*(R2 + D2)) <= (qd - qn)/qd <= D2 / (2*R2)` for RMSNorm radial base-logit shrinkage. -/ +theorem Mcproof.Spec.rmsnorm_rational_shrinkage_sandwich + (R2 D2 qn qd : Int) + (hR : 0 < R2) (hD : 0 ≤ D2) (hqn : 0 < qn) (hqd : 0 < qd) + (heq : (R2 + D2) * (qn * qn) = R2 * (qd * qd)) : + shrinkageSandwichCheck R2 D2 qn qd = true ∧ + D2 * qd ≤ 2 * (R2 + D2) * (qd - qn) ∧ + 2 * R2 * (qd - qn) ≤ D2 * qd := sorry diff --git a/lean/Mcproof/TransitionBounds.lean b/lean/Mcproof/TransitionBounds.lean new file mode 100644 index 0000000..eec3076 --- /dev/null +++ b/lean/Mcproof/TransitionBounds.lean @@ -0,0 +1,104 @@ +/- + Domain-general discrete state-transition and optimization lower-bound kernels (mcproof v0.2): + 1. Quotient graph homomorphism heuristics (`quotientHeuristic`) over `ValidPath`. + 2. Pre-funded monotone potential repair (`prefundedVal`, `prefundedHeuristic`). + 3. Non-negative operator-counting / Farkas LP dual lower bounds (`dualObjective`, `dualFloorNat`). + 4. Cyclic mode-switch metric (`cyclicSwitchDist`) for multi-mode controllers and schedulers. + 5. Two-species conservation & irreversible-harvest intact-supply potentials (`countTrue`, `intactSupply`). + 6. Time-expanded vertex-split unit-capacity flow & min-cut horizon bounds (`vertexArcFlow`, `tokensInCut`, `horizonCutCapacity`). + Stock Lean 4.35.0 Init only: zero floats, zero custom axioms, zero escape hatches. +-/ + +namespace Mcproof.TransitionBounds + +/-- Valid finite action sequence in a deterministic partial transition system (`none` = invalid/sink). -/ +inductive ValidPath {S A : Type} (step : S → A → Option S) : S → List A → S → Prop where + | nil (s : S) : ValidPath step s [] s + | cons {s s' g : S} {a : A} {as : List A} : + step s a = some s' → ValidPath step s' as g → ValidPath step s (a :: as) g + +/-- Pullback of a quotient heuristic `hQ : Q -> Nat` along state projection `pi : S -> Q`. -/ +def quotientHeuristic {S Q : Type} (pi : S → Q) (hQ : Q → Nat) (s : S) : Nat := + hQ (pi s) + +/-- Exact `Nat` pre-funded potential `max(0, hVal - c * rVal)` via truncating natural subtraction. -/ +def prefundedVal (hVal c rVal : Nat) : Nat := + hVal - c * rVal + +/-- State-level pre-funded monotone resource potential `h_hat(s) = h(s) - c * r(s)` in `Nat`. -/ +def prefundedHeuristic {S : Type} (h : S → Nat) (c : Nat) (r : S → Nat) (s : S) : Nat := + prefundedVal (h s) c (r s) + +/-- Weighted operator-counting / landmark inequality `demand <= sum_{a in plan} coeff(a)`. -/ +structure DualConstraint (A : Type) where + weight : Nat + demand : Nat + coeff : A → Nat + +/-- Additive sum of an action valuation `f : A -> Nat` along an action sequence `acts`. -/ +def planSum {A : Type} (f : A → Nat) : List A → Nat + | [] => 0 + | a :: as => f a + planSum f as + +/-- Scaled dual objective `sum_{i} weight_i * demand_i` across a constraint list. -/ +def dualObjective {A : Type} : List (DualConstraint A) → Nat + | [] => 0 + | c :: cs => c.weight * c.demand + dualObjective cs + +/-- Total dual load `sum_{i} weight_i * coeff_i(a)` charged to a single action `a`. -/ +def actionDualLoad {A : Type} : List (DualConstraint A) → A → Nat + | [], _ => 0 + | c :: cs, a => c.weight * c.coeff a + actionDualLoad cs a + +/-- Executable integer dual objective over concrete `(weight, demand)` pairs. -/ +def dualObjectivePairs : List (Nat × Nat) → Nat + | [] => 0 + | (w, d) :: rest => w * d + dualObjectivePairs rest + +/-- Executable division-free rational floor `floor((sum_i w_i * d_i) / scale)` for positive `scale`. -/ +def dualFloorNat (scale : Nat) (pairs : List (Nat × Nat)) : Nat := + if scale == 0 then 0 else dualObjectivePairs pairs / scale + +/-- Unwrapped cyclic mode-switch step `(b + k - a) % k` for pre-reduced modes `< k`. -/ +def cyclicSwitchTax (k a b : Nat) : Nat := + (b + k - a) % k + +/-- Directed cyclic mode-switch distance `((b % k) + k - (a % k)) % k` on a `k`-mode cycle. -/ +def cyclicSwitchDist (k a b : Nat) : Nat := + if k == 0 then 0 else cyclicSwitchTax k (a % k) (b % k) + +/-- Cardinality of elements in `xs` satisfying boolean predicate `p`. -/ +def countTrue {X : Type} (p : X → Bool) : List X → Nat + | [] => 0 + | x :: xs => (if p x then 1 else 0) + countTrue p xs + +/-- Remaining intact supply on tracked site list `I` in state `s`. -/ +def intactSupply {S X : Type} (I : List X) (intact : S → X → Bool) (s : S) : Nat := + countTrue (intact s) I + +/-- Unit vertex-split internal arc flow `v_in(t) -> v_out(t)` (`0` or `1`) induced by occupancy `occ`. -/ +def vertexArcFlow {V : Type} (occ : Nat → V → Bool) (t : Nat) (v : V) : Nat := + if occ t v then 1 else 0 + +/-- Number of occupied vertices inside cut `cut_t` across vertex list `vertices`. -/ +def tokensInCut {V : Type} (vertices : List V) (occ_t : V → Bool) (cut_t : V → Bool) : Nat := + countTrue (fun v => occ_t v && cut_t v) vertices + +/-- Cumulative cut capacity `sum_{t = 0}^{T - 1} cutCap(t)` across horizon `[0, T)`. -/ +def horizonCutCapacity (cutCap : Nat → Nat) : Nat → Nat + | 0 => 0 + | t + 1 => horizonCutCapacity cutCap t + cutCap t + +/-- Remaining target deficit `max(0, K - mVal)` in `Nat`. -/ +def deficitNat (K mVal : Nat) : Nat := + K - mVal + +/-- Composed quotient-plus-deficit heuristic `hQ(pi(s)) + (K - m(s))` for partitioned action sets. -/ +def composedDeficitHeuristic {S Q : Type} (pi : S → Q) (hQ : Q → Nat) (K : Nat) (m : S → Nat) (s : S) : Nat := + quotientHeuristic pi hQ s + deficitNat K (m s) + +/-- Exact integer ceiling `ceil(num / scale)` for positive `scale` (`0` when `scale == 0`). -/ +def ceilDivNat (num scale : Nat) : Nat := + if scale == 0 then 0 else (num + scale - 1) / scale + +end Mcproof.TransitionBounds diff --git a/lean/README.md b/lean/README.md index 1c4f9ae..de0c9ea 100644 --- a/lean/README.md +++ b/lean/README.md @@ -1,6 +1,6 @@ # mcproof Lean 4 Reference Package -`mcproof` in `lean/` is a stock-`Init` Lean 4.35.0 library and executable that checks bounded clamping, integer Bezout witnesses, division-free rational order, and path-segment containment. Proof engineers and automated verification harnesses use it to validate multi-kernel theorem checking and anti-bypass rules under the [Proven Commons Verified Bar](https://github.com/Proven-Commons/mission/blob/main/playbook/verified-bar.md). +`mcproof` in `lean/` is a stock-`Init` Lean 4.35.0 library and executable checking bounded clamping, Bezout witnesses, rational order, path containment, and discrete state-transition lower-bound soundness. Engineers use it to validate multi-kernel theorem checking under the [Proven Commons Verified Bar](https://github.com/Proven-Commons/mission/blob/main/playbook/verified-bar.md). Status: in progress. Independent human specification review has not been recorded yet, so this package is not verified. @@ -16,17 +16,38 @@ Status: in progress. Independent human specification review has not been recorde | Every fraction with `0 < d` passes `ratLeCross n d n d`. | `Mcproof.Spec.ratLeCross_refl_pos` | | Accepted path lists from `cleanSegments` are non-empty and exclude `.`, `..`, `/`, and NUL. | `Mcproof.Spec.cleanSegments_safe` | | A single valid segment `[s]` returns `some [s]` from `cleanSegments`. | `Mcproof.Spec.cleanSegments_singleton` | +| Quotient homomorphism pullback `hQ (pi s)` is globally 1-step consistent on full states. | `Mcproof.Spec.quotientHeuristic_step_consistent` | +| Quotient homomorphism pullback bounds every `ValidPath` reaching a goal state. | `Mcproof.Spec.quotientHeuristic_admissible` | +| Pre-funded resource potential `h(s) - c * r(s)` preserves `0` on goals and admissibility. | `Mcproof.Spec.prefundedHeuristic_admissible_and_goal` | +| Pre-funded potential restores universal `h_hat(s) <= h_hat(s') + 1` in `Nat`. | `Mcproof.Spec.prefundedHeuristic_step_consistent` | +| Concrete `12 -> 10` witness repairs to `11 -> 10` under `c = 1, r : 1 -> 0`. | `Mcproof.Spec.prefundedVal_bounded_drop_witness` | +| Non-negative operator-counting / Farkas dual bounds `scale * planSum cost acts`. | `Mcproof.Spec.operatorCounting_dual_scaled_sound` | +| Division-free integer dual floor `dualObjective cs / scale` is admissible for `0 < scale`. | `Mcproof.Spec.operatorCounting_dual_div_admissible` | +| Executable `dualFloorNat` bounds `planCost` and checks `dualFloorNat 2 [(2,9),(1,8)] = 13`. | `Mcproof.Spec.dualFloorNat_sound_and_witness` | +| Cyclic mode-switch tax `(c + k - a) % k` satisfies triangle inequality for `a, b, c < k`. | `Mcproof.Spec.cyclicSwitchTax_triangle` | +| Unrestricted `cyclicSwitchDist` satisfies zero self-tax, triangle inequality, and `k = 6` witnesses. | `Mcproof.Spec.cyclicSwitchDist_metric_and_witness` | +| Two-species conservation and irreversible harvests fund global pre-funded potential repair. | `Mcproof.Spec.conservation_harvest_potential_repair` | +| Vertex-split unit-capacity flows bound horizon demand by cumulative min-cut capacity. | `Mcproof.Spec.vertexSplit_unit_flow_horizon_bound` | +| Zero-self-loop quotient + monotone deficit composition preserves 1-step consistency and admissibility. | `Mcproof.Spec.zeroSelfLoop_deficit_composition` | +| Exact rational ceiling `ceilDivNat` preserves `0` on goals, 1-step consistency, and admissibility. | `Mcproof.Spec.ceilDiv_step_consistent_and_admissible` | +| Division-free RMSNorm tangent projection `tangentProjScaled h delta` is orthogonal to `h`. | `Mcproof.Spec.rmsnorm_tangent_orthogonal` | +| Pairwise-orthogonal tangent adapter stacks inflate squared norm additively. | `Mcproof.Spec.rmsnorm_orthogonal_stack_norm_inflation` | +| Balanced `L`-layer split cuts cumulative quadratic radial shrinkage by exact factor `L`. | `Mcproof.Spec.multilayer_quadratic_shrinkage_reduction` | +| Elementwise gate inner product equals `diag(g^2)` metric with `[2, 1]` counterexample. | `Mcproof.Spec.swiglu_gate_metric_and_counterexample` | +| Hadamard adjoint projection orthogonal to `g ⊙ (W_down^T h_0)` zeroes radial inner product. | `Mcproof.Spec.swiglu_radial_null_orthogonality` | +| Orthogonal `[0, 32]` cross-talk keeps `100%` scalar projection while injecting `1024x` drift energy. | `Mcproof.Spec.scalar_projection_retention_illusion` | +| RMSNorm radial base-logit shrinkage lies in exact rational sandwich `[D2/(2*(R2+D2)), D2/(2*R2)]`. | `Mcproof.Spec.rmsnorm_rational_shrinkage_sandwich` | ## What is not proved - Command-line string parsing and standard-output formatting in `Main.lean` are covered by differential tests rather than Lean theorems. -- Operating-system symlink resolution on live filesystems sits outside lexical segment checks. -- Execution latency and memory usage are not proved. +- Whether a concrete transition-system implementation satisfies the quotient-edge or dual-load hypotheses on all reachable transitions must be verified by exhaustive edge or certificate checks. +- Operating-system symlink resolution, execution latency, and memory usage sit outside these theorems. ## What you must trust - The Lean 4.35.0 kernel and the five bundled external kernels (`leanchecker-paranoid`, `lean4lean`, `nanoda`, `con-leche`, `con-ron`). -- The Lean compiler and C runtime that lower `Mcproof.lean` and `Main.lean` into native machine code. +- The Lean compiler and C runtime that lower `Mcproof.lean`, `Mcproof/TransitionBounds.lean`, and `Main.lean` into native machine code. - The host hardware, operating system, and `bubblewrap` sandbox on Linux hosts. - Python 3 exact `int` and `fractions.Fraction` arithmetic in the differential test oracle. diff --git a/lean/SPEC.md b/lean/SPEC.md index 887878d..e0a1056 100644 --- a/lean/SPEC.md +++ b/lean/SPEC.md @@ -1,6 +1,6 @@ # Specification: mcproof Lean Reference Library -`mcproof` in `lean/` is the Lean 4 reference package for the Formal Verification MDE. It defines four pure kernels over natural numbers, arbitrary-precision integers, exact rational pairs, and relative path segments, paired with challenge and proof modules checked by `lake comparator --paranoid`. +`mcproof` in `lean/` is the domain-general Lean 4.35.0 reference package for the Formal Verification MDE. It defines pure kernels for natural clamping, exact Bezout witnesses, division-free rational order, path containment, quotient state-transition heuristics, pre-funded monotone potential repair, and operator-counting LP dual bounds, checked by `lake comparator --paranoid`. ## Scope @@ -8,6 +8,11 @@ 2. `Mcproof.bezoutCheck a b x y g` returns `true` if and only if `a * x + b * y = g` over exact integers. 3. `Mcproof.ratLeCross n1 d1 n2 d2` checks `n1 / d1 <= n2 / d2` for positive denominators via exact integer cross-multiplication `n1 * d2 <= n2 * d1`. 4. `Mcproof.cleanSegments segs` accepts a non-empty list of segments only when every segment is non-empty, neither `.` nor `..`, and free of `/` and NUL bytes. +5. `Mcproof.TransitionBounds.quotientHeuristic pi hQ` pulls back a 1-step-consistent quotient heuristic `hQ` to full states over `ValidPath`. +6. `Mcproof.TransitionBounds.prefundedVal h c r` and `prefundedHeuristic` subtract monotone resource credit `c * r` in `Nat` to restore universal 1-step consistency under bounded multi-step heuristic drops. +7. `Mcproof.TransitionBounds.dualObjective`, `actionDualLoad`, and `dualFloorNat` compute division-free integer lower bounds from non-negative operator-counting / Farkas dual constraints. +8. `Mcproof.TransitionBounds.cyclicSwitchTax`, `cyclicSwitchDist`, `intactSupply`, `vertexArcFlow`, `tokensInCut`, `horizonCutCapacity`, `composedDeficitHeuristic`, and `ceilDivNat` formalize cyclic mode-switch metrics, two-species conservation potentials, vertex-split flow horizon bounds, composed deficits, and exact ceilings. +9. `Mcproof.AdapterBounds.tangentProjScaled`, `vecDot`, `vecNormSq`, `vecAdd`, `vecSub`, `vecHadamard`, `vecSqDiag`, `sumNat`, `sumSqNat`, and `shrinkageSandwichCheck` formalize exact integer RMSNorm tangent projection, multi-layer `1/L` quadratic radial-shrinkage reduction, SwiGLU gate metrics, and rational shrinkage bounds. ## What must always hold @@ -21,6 +26,27 @@ | Every fraction with `0 < d` satisfies `ratLeCross n d n d = true`. | `Mcproof.Spec.ratLeCross_refl_pos` | | Accepted segment lists are non-empty and free of `.`, `..`, `/`, and NUL. | `Mcproof.Spec.cleanSegments_safe` | | Every single safe segment `[s]` is accepted as `some [s]`. | `Mcproof.Spec.cleanSegments_singleton` | +| Quotient homomorphism heuristics satisfy `h(s) <= h(s') + 1` on every valid transition. | `Mcproof.Spec.quotientHeuristic_step_consistent` | +| Quotient homomorphism heuristics bound the length of every valid goal-reaching path. | `Mcproof.Spec.quotientHeuristic_admissible` | +| Pre-funded potentials preserve `h_hat(g) = 0` and inherit admissibility `h_hat(s) <= len`. | `Mcproof.Spec.prefundedHeuristic_admissible_and_goal` | +| Monotone resource `r(s') <= r(s)` repairs `1 + c * (r(s) - r(s'))` drops to `h_hat(s) <= h_hat(s') + 1`. | `Mcproof.Spec.prefundedHeuristic_step_consistent` | +| Concrete `12 -> 10` drop repairs to `11 -> 10` under `c = 1, r : 1 -> 0`. | `Mcproof.Spec.prefundedVal_bounded_drop_witness` | +| Non-negative operator-counting dual weights satisfy `dualObjective cs <= scale * planSum cost acts`. | `Mcproof.Spec.operatorCounting_dual_scaled_sound` | +| For `0 < scale`, `(dualObjective cs) / scale <= planSum cost acts`. | `Mcproof.Spec.operatorCounting_dual_div_admissible` | +| Executable `dualFloorNat` bounds `planCost` and evaluates `dualFloorNat 2 [(2,9),(1,8)] = 13`. | `Mcproof.Spec.dualFloorNat_sound_and_witness` | +| Cyclic mode-switch tax `(c + k - a) % k` satisfies triangle inequality for modes `< k`. | `Mcproof.Spec.cyclicSwitchTax_triangle` | +| Unrestricted `cyclicSwitchDist` satisfies identity, triangle inequality, and `k = 6` witnesses. | `Mcproof.Spec.cyclicSwitchDist_metric_and_witness` | +| Two-species conservation and irreversible harvests ensure global potential consistency and admissibility. | `Mcproof.Spec.conservation_harvest_potential_repair` | +| Collision-free trajectories induce unit vertex-split flows bounded by cumulative min-cut capacity. | `Mcproof.Spec.vertexSplit_unit_flow_horizon_bound` | +| Zero-self-loop quotient heuristics compose with monotone target deficits while preserving 1-step consistency. | `Mcproof.Spec.zeroSelfLoop_deficit_composition` | +| Exact rational ceiling `ceilDivNat` preserves `0` on goals, 1-step consistency, and admissibility. | `Mcproof.Spec.ceilDiv_step_consistent_and_admissible` | +| Division-free RMSNorm tangent projection `tangentProjScaled h delta` is orthogonal to `h`. | `Mcproof.Spec.rmsnorm_tangent_orthogonal` | +| Pairwise-orthogonal tangent adapter stacks inflate squared norm additively. | `Mcproof.Spec.rmsnorm_orthogonal_stack_norm_inflation` | +| Splitting total impulse `D = L * d` across `L` layers cuts quadratic shrinkage by factor `L`. | `Mcproof.Spec.multilayer_quadratic_shrinkage_reduction` | +| Elementwise gate inner product equals `diag(g^2)` metric and breaks naive `X X^T` orthogonality. | `Mcproof.Spec.swiglu_gate_metric_and_counterexample` | +| Projecting up-residuals orthogonal to `g ⊙ (W_down^T h_0)` zeroes radial post-gate inner product. | `Mcproof.Spec.swiglu_radial_null_orthogonality` | +| Orthogonal sibling cross-talk `[0, 32]` keeps `100%` scalar projection while adding `1024x` drift energy. | `Mcproof.Spec.scalar_projection_retention_illusion` | +| RMSNorm radial base-logit shrinkage lies inside `[D2 / (2*(R2 + D2)), D2 / (2*R2)]`. | `Mcproof.Spec.rmsnorm_rational_shrinkage_sandwich` | ## Oracle diff --git a/lean/bar.toml b/lean/bar.toml index 81deb76..bc24d54 100644 --- a/lean/bar.toml +++ b/lean/bar.toml @@ -12,7 +12,41 @@ code = [ "Mcproof.bezoutCheck", "Mcproof.ratLeCross", "Mcproof.isSafeSegment", - "Mcproof.cleanSegments" + "Mcproof.cleanSegments", + "Mcproof.TransitionBounds.ValidPath", + "Mcproof.TransitionBounds.quotientHeuristic", + "Mcproof.TransitionBounds.prefundedVal", + "Mcproof.TransitionBounds.prefundedHeuristic", + "Mcproof.TransitionBounds.DualConstraint", + "Mcproof.TransitionBounds.DualConstraint.weight", + "Mcproof.TransitionBounds.DualConstraint.demand", + "Mcproof.TransitionBounds.DualConstraint.coeff", + "Mcproof.TransitionBounds.planSum", + "Mcproof.TransitionBounds.dualObjective", + "Mcproof.TransitionBounds.actionDualLoad", + "Mcproof.TransitionBounds.dualObjectivePairs", + "Mcproof.TransitionBounds.dualFloorNat", + "Mcproof.TransitionBounds.cyclicSwitchTax", + "Mcproof.TransitionBounds.cyclicSwitchDist", + "Mcproof.TransitionBounds.countTrue", + "Mcproof.TransitionBounds.intactSupply", + "Mcproof.TransitionBounds.vertexArcFlow", + "Mcproof.TransitionBounds.tokensInCut", + "Mcproof.TransitionBounds.horizonCutCapacity", + "Mcproof.TransitionBounds.deficitNat", + "Mcproof.TransitionBounds.composedDeficitHeuristic", + "Mcproof.TransitionBounds.ceilDivNat", + "Mcproof.AdapterBounds.vecDot", + "Mcproof.AdapterBounds.vecNormSq", + "Mcproof.AdapterBounds.vecAdd", + "Mcproof.AdapterBounds.vecSub", + "Mcproof.AdapterBounds.vecScale", + "Mcproof.AdapterBounds.vecHadamard", + "Mcproof.AdapterBounds.vecSqDiag", + "Mcproof.AdapterBounds.tangentProjScaled", + "Mcproof.AdapterBounds.sumNat", + "Mcproof.AdapterBounds.sumSqNat", + "Mcproof.AdapterBounds.shrinkageSandwichCheck" ] [tests] diff --git a/lean/comparator.json b/lean/comparator.json index 655ce43..dfdc8dd 100644 --- a/lean/comparator.json +++ b/lean/comparator.json @@ -9,7 +9,28 @@ "Mcproof.Spec.ratLeCross_sound", "Mcproof.Spec.ratLeCross_refl_pos", "Mcproof.Spec.cleanSegments_safe", - "Mcproof.Spec.cleanSegments_singleton" + "Mcproof.Spec.cleanSegments_singleton", + "Mcproof.Spec.quotientHeuristic_step_consistent", + "Mcproof.Spec.quotientHeuristic_admissible", + "Mcproof.Spec.prefundedHeuristic_admissible_and_goal", + "Mcproof.Spec.prefundedHeuristic_step_consistent", + "Mcproof.Spec.prefundedVal_bounded_drop_witness", + "Mcproof.Spec.operatorCounting_dual_scaled_sound", + "Mcproof.Spec.operatorCounting_dual_div_admissible", + "Mcproof.Spec.dualFloorNat_sound_and_witness", + "Mcproof.Spec.cyclicSwitchTax_triangle", + "Mcproof.Spec.cyclicSwitchDist_metric_and_witness", + "Mcproof.Spec.conservation_harvest_potential_repair", + "Mcproof.Spec.vertexSplit_unit_flow_horizon_bound", + "Mcproof.Spec.zeroSelfLoop_deficit_composition", + "Mcproof.Spec.ceilDiv_step_consistent_and_admissible", + "Mcproof.Spec.rmsnorm_tangent_orthogonal", + "Mcproof.Spec.rmsnorm_orthogonal_stack_norm_inflation", + "Mcproof.Spec.multilayer_quadratic_shrinkage_reduction", + "Mcproof.Spec.swiglu_gate_metric_and_counterexample", + "Mcproof.Spec.swiglu_radial_null_orthogonality", + "Mcproof.Spec.scalar_projection_retention_illusion", + "Mcproof.Spec.rmsnorm_rational_shrinkage_sandwich" ], "permitted_axioms": [ "propext", diff --git a/lean/tests/TEST-RESULTS.md b/lean/tests/TEST-RESULTS.md index 816a444..2b9b67f 100644 --- a/lean/tests/TEST-RESULTS.md +++ b/lean/tests/TEST-RESULTS.md @@ -1,6 +1,6 @@ # Differential Test Record -- Oracle: Python 3.14 int and fractions.Fraction +- Oracle: Python 3 int and fractions.Fraction - Seed: 20261119 -- Inputs: 190 +- Inputs: 370 - Failures: 0 diff --git a/lean/tests/run_tests.py b/lean/tests/run_tests.py index 6963f63..a872592 100644 --- a/lean/tests/run_tests.py +++ b/lean/tests/run_tests.py @@ -7,7 +7,6 @@ from pathlib import Path import random import subprocess -import sys ROOT = Path(__file__).resolve().parent.parent SEED = 20261119 @@ -59,6 +58,29 @@ def py_clean_segments(segs: list[str]) -> str: return "OK:" + "/".join(segs) +def py_prefund(h_val: int, c_val: int, r_val: int) -> int: + return max(0, h_val - c_val * r_val) + + +def py_dual_floor(scale: int, pairs: list[tuple[int, int]]) -> int: + if scale == 0: + return 0 + num = sum(w * d for w, d in pairs) + return int(Fraction(num, scale) // 1) + + +def py_cyclic_dist(k: int, a: int, b: int) -> int: + if k == 0: + return 0 + return ((b % k) + k - (a % k)) % k + + +def py_ceil_div(num: int, scale: int) -> int: + if scale == 0: + return 0 + return -(-num // scale) + + def main() -> int: exe = find_binary() rng = random.Random(SEED) @@ -119,14 +141,80 @@ def main() -> int: if got != exp: failures += 1 + # Pre-funded monotone potential differential checks (including zero clamping boundary) + prefund_cases = [(12, 1, 1), (10, 1, 0), (5, 2, 4), (0, 3, 2)] + [ + (rng.randint(0, 120), rng.randint(0, 6), rng.randint(0, 25)) for _ in range(36) + ] + for h_val, c_val, r_val in prefund_cases: + got = int(call_cli(exe, "prefund", str(h_val), str(c_val), str(r_val))) + exp = py_prefund(h_val, c_val, r_val) + total_inputs += 1 + if got != exp: + failures += 1 + + # Operator-counting / Farkas dual floor differential checks + for idx in range(30): + scale = 0 if idx == 0 else rng.randint(1, 12) + k_len = rng.randint(0, 6) + pairs = [(rng.randint(0, 15), rng.randint(0, 30)) for _ in range(k_len)] + flat_args = [x for w, d in pairs for x in (str(w), str(d))] + got = int(call_cli(exe, "dualfloor", str(scale), *flat_args)) + exp = py_dual_floor(scale, pairs) + total_inputs += 1 + if got != exp: + failures += 1 + + # Cyclic mode-switch metric differential & triangle checks + for idx in range(40): + k_val = 0 if idx == 0 else rng.randint(1, 16) + a_val = rng.randint(0, 50) + b_val = rng.randint(0, 50) + got = int(call_cli(exe, "cyclicdist", str(k_val), str(a_val), str(b_val))) + exp = py_cyclic_dist(k_val, a_val, b_val) + total_inputs += 1 + if got != exp: + failures += 1 + + # Rational ceiling differential checks + for idx in range(30): + scale_c = 0 if idx == 0 else rng.randint(1, 20) + num_c = 0 if idx == 1 else rng.randint(0, 250) + got = int(call_cli(exe, "ceildiv", str(num_c), str(scale_c))) + exp = py_ceil_div(num_c, scale_c) + total_inputs += 1 + if got != exp: + failures += 1 + + # RMSNorm tangent projection differential & orthogonality checks + for _ in range(25): + h1, h2 = rng.randint(-30, 30), rng.randint(-30, 30) + d1, d2 = rng.randint(-30, 30), rng.randint(-30, 30) + got_parts = [int(x) for x in call_cli(exe, "tangentproj", str(h1), str(h2), str(d1), str(d2)).split(",")] + nh = h1 * h1 + h2 * h2 + hd = h1 * d1 + h2 * d2 + exp_parts = [nh * d1 - hd * h1, nh * d2 - hd * h2] + total_inputs += 1 + if got_parts != exp_parts or (h1 * got_parts[0] + h2 * got_parts[1]) != 0: + failures += 1 + + # Multi-layer sumSqNat differential checks + for _ in range(15): + k_layers = rng.randint(0, 6) + ds = [rng.randint(0, 20) for _ in range(k_layers)] + got_sq = int(call_cli(exe, "sumsq", *[str(v) for v in ds])) + exp_sq = sum(v * v for v in ds) + total_inputs += 1 + if got_sq != exp_sq: + failures += 1 + results_md = ( "# Differential Test Record\n\n" - f"- Oracle: Python {sys.version_info.major}.{sys.version_info.minor} int and fractions.Fraction\n" + "- Oracle: Python 3 int and fractions.Fraction\n" f"- Seed: {SEED}\n" f"- Inputs: {total_inputs}\n" f"- Failures: {failures}\n" ) - (ROOT / "tests" / "TEST-RESULTS.md").write_text(results_md, encoding="utf-8") + (ROOT / "tests" / "TEST-RESULTS.md").write_text(results_md, encoding="utf-8", newline="\n") if failures != 0: raise RuntimeError(f"{failures} differential test failure(s) across {total_inputs} inputs") return 0 diff --git a/spec/lineage.json b/spec/lineage.json index 97d9250..aa592a0 100644 --- a/spec/lineage.json +++ b/spec/lineage.json @@ -21,6 +21,9 @@ "plane1_spec_token_stats": { "char_div4_estimate": 26690, "cl100k_base_tokens": 30873, + "cl100k_tokenizer": "tiktoken/0.14.0:cl100k_base", + "qwen35_tokenizer": "Qwen/Qwen3.5-4B (tokenizer.json sha256:5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42)", + "qwen35_tokens": 32216, "utf8_bytes": 106763, "words": 15791 }, @@ -33,7 +36,7 @@ }, "source_lineage": { "contested_heads": [], - "corpus_tree_sha256": "5bdb7f30363b956e82947f668f98c48f99876e87fb5ed4f785b873e4f4f98305", + "corpus_tree_sha256": "93624aae09eea99c3d35ebfa2e9acde1f9ac9c675167461652486850f206b642", "domain_commits": { "corpus": "7b22e542f0db0f3013ab1439cf5d1947676a5083", "formal-verification": "49beff2a0185cfa4032170b614a543c0d6754149", @@ -41,10 +44,12 @@ } }, "tracked_plane1_files": { - "lean/Mcproof.lean": "e27f69bce4f77b2b0fc308880a84e8e6c0710b578532167f1e27905595f0722c", - "lean/Mcproof/Proofs.lean": "27d1ae954c2e3ed1b76df409e0f59f41eca9c1ab13e6f4263d9a88a2ef8c6731", - "lean/Mcproof/Spec.lean": "0e4f403806727a088382477a3eddf8f7d67c0377a3e7391af862f7da1a67cca5", - "lean/comparator.json": "223b808dc8266557829fa21d515a0b0bbdf3f28a7242b03a23c1d806520d34e8", + "lean/Mcproof.lean": "e629ec64b3f433ad01d39efc5c61701a4aec10c612a0786a9bdaeeb1805bf03b", + "lean/Mcproof/AdapterBounds.lean": "d1152e3b37dd9fb078bae07419e92b3aeec9ddfae49c6ccd57fb6b0b8d86519a", + "lean/Mcproof/Proofs.lean": "09bbd22fe5b6fcb5fd1a7899fdb156f0b49f4953a49cacec10af2ec5042063df", + "lean/Mcproof/Spec.lean": "9baf38612b0443614b086efaf214ec7b6abbcf196019090e51d9173fc8a11304", + "lean/Mcproof/TransitionBounds.lean": "5d6c3a4efefc2c84b034ebabadb6e59965932158d1964b7d27e8947d056aaf6b", + "lean/comparator.json": "47121f42a035fd2f023a06bcc88c8580d9a6f8e06c51cf9625d2bbb0a630a9f1", "lean/lakefile.toml": "8540e8392738181588c16176e164496608dc5c89e1d359150d991d6f18c3d0ec", "lean/lean-toolchain": "ca5f1f601d34c49079e38d7e7041eb0f2a9c227fbc1df0f404820069a052330c", "schemas/dataset-record.v0.1.schema.json": "5891b56778196ac7d9a2ea30c5d673c2f7a3f5daa1da0da34c91518a482cbbea", diff --git a/src/mcproof/__init__.py b/src/mcproof/__init__.py index ac6ccfd..5215eeb 100644 --- a/src/mcproof/__init__.py +++ b/src/mcproof/__init__.py @@ -2,12 +2,28 @@ from __future__ import annotations +from .domain_protocol import ( + BudgetLedger, + DomainDiagnostic, + DomainGradeResult, + DomainOracle, + FormalVerificationDomainOracle, + audit_portfolio_bounds, + audit_transition_certificate, + loads_strict_certificate_json, + run_restricted_sandbox_cell, + verify_dual_product_parity, + verify_minimal_goal_stopping, + verify_rational_lp_certificate, +) from .lean_harness import ( LeanVerificationResult, audit_weak_statements, lint_lean4_source, + scrub_challenge_artifacts, verify_lean4_project_comparator, verify_lean4_source, + verify_pinned_lean_defs, ) from .lineage import ( CANONICAL_ZONE_MANIFEST_ID, @@ -40,11 +56,28 @@ run_soko_integrity_and_evidence_oracle, run_verdicts_oracle, ) +from .repair_loop import ( + compute_candidate_state_hash, + compute_failure_signature, + default_inference_oracle, + format_diagnostics, + make_llama_server_generator, + retrieve_plane1_excerpt, + retrieve_plane1_sections, + run_repair_item, + run_repair_split, + sanitize_oracle_diagnostics, +) -__version__ = "0.1.0" +__version__ = "0.2.0" __all__ = [ + "BudgetLedger", "CANONICAL_ZONE_MANIFEST_ID", + "DomainDiagnostic", + "DomainGradeResult", + "DomainOracle", + "FormalVerificationDomainOracle", "KILL_FLOOR_DEN", "KILL_FLOOR_NUM", "LeanVerificationResult", @@ -53,11 +86,21 @@ "OracleReceipt", "StaleLineageError", "__version__", + "audit_portfolio_bounds", + "audit_transition_certificate", "audit_weak_statements", "build_lineage_manifest", + "compute_candidate_state_hash", + "compute_failure_signature", + "default_inference_oracle", "find_forbidden_control_bytes", + "format_diagnostics", "grade_family_completion", "lint_lean4_source", + "loads_strict_certificate_json", + "make_llama_server_generator", + "retrieve_plane1_excerpt", + "retrieve_plane1_sections", "run_all_reference_selftests", "run_checker_oracle", "run_claim_hygiene_oracle", @@ -69,9 +112,18 @@ "run_mission_bar_oracle", "run_prose_and_mixed_zone_oracle", "run_python_strict", + "run_repair_item", + "run_repair_split", + "run_restricted_sandbox_cell", "run_soko_integrity_and_evidence_oracle", "run_verdicts_oracle", + "sanitize_oracle_diagnostics", + "scrub_challenge_artifacts", + "verify_dual_product_parity", "verify_lean4_project_comparator", "verify_lean4_source", "verify_lineage_manifest", + "verify_minimal_goal_stopping", + "verify_pinned_lean_defs", + "verify_rational_lp_certificate", ] diff --git a/src/mcproof/cli.py b/src/mcproof/cli.py index 785c2fe..cf7f677 100644 --- a/src/mcproof/cli.py +++ b/src/mcproof/cli.py @@ -9,6 +9,11 @@ import sys from typing import Any, Sequence +from .domain_protocol import ( + audit_transition_certificate, + loads_strict_certificate_json, + run_restricted_sandbox_cell, +) from .lineage import build_lineage_manifest, verify_lineage_manifest from .oracles import ( OracleReceipt, @@ -25,6 +30,11 @@ run_verdicts_oracle, ) from .paths import repo_root +from .repair_loop import ( + make_llama_server_generator, + retrieve_plane1_excerpt, + run_repair_split, +) def _read_text_or_stdin(path_arg: str) -> str: @@ -119,6 +129,30 @@ def build_parser() -> argparse.ArgumentParser: s_grd.add_argument("--expected-tier", default=None, help="Expected claim tier") s_grd.add_argument("--fixture-json", default=None, help="Optional JSON fixture with generator_source / valid_cases / planted_mutations") + # repair + s_rep = sub.add_parser("repair", help="Run targeted <=1500-token Plane 1 retrieval or closed-loop k-turn repair") + s_rep.add_argument("--dataset", default=None, help="JSONL dataset split path") + s_rep.add_argument("--server-url", default="http://127.0.0.1:18080", help="llama-server base URL") + s_rep.add_argument("--lora-json", default="[]", help="JSON list of llama-server LoRA specs") + s_rep.add_argument("--max-turns", type=int, default=4, help="Maximum repair turns k (default: 4)") + s_rep.add_argument("--max-predict", type=int, default=768, help="Max completion tokens per turn") + s_rep.add_argument("--retrieval-max-tokens", type=int, default=1250, help="Plane 1 excerpt cap (hard max 1500)") + s_rep.add_argument("--no-plane1", action="store_true", help="Disable Plane 1 excerpt retrieval") + s_rep.add_argument("--checkpoint-jsonl", default=None, help="Crash-safe JSONL checkpoint path") + s_rep.add_argument("--output-json", default=None, help="Output summary JSON path") + s_rep.add_argument("--retrieve-only", action="store_true", help="Only retrieve and verify <=1500-token Plane 1 excerpt") + s_rep.add_argument("--family", default="A", choices=["A", "B", "C", "D", "a", "b", "c", "d"], help="Family for --retrieve-only") + s_rep.add_argument("--prompt", default="", help="Prompt text for --retrieve-only") + + # audit-cert + s_crt = sub.add_parser("audit-cert", help="Audit a state-transition / lower-bound certificate JSON for structural soundness") + s_crt.add_argument("cert_file", help="Path to certificate JSON file or '-' for stdin") + + # sandbox-cell + s_sbx = sub.add_parser("sandbox-cell", help="Execute a 4-second network-free, filesystem-isolated Python micro-computation cell") + s_sbx.add_argument("code_file", help="Path to Python snippet or '-' for stdin") + s_sbx.add_argument("--timeout", type=float, default=4.0, help="Timeout in seconds (hard-capped at 4.0s)") + # selftest s_self = sub.add_parser("selftest", help="Run full Tier 2 oracle and Plane 1 lineage self-test suite") s_self.add_argument("--allow-unsandboxed", action="store_true", help="Explicitly permit comparator `--inadvisably-no-sandbox` when Linux bwrap is unavailable") @@ -234,7 +268,7 @@ def _dispatch(args: argparse.Namespace) -> int: manifest = build_lineage_manifest(r_root) out_path = r_root / "spec" / "lineage.json" out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + out_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8", newline="\n") rep = verify_lineage_manifest(r_root, require_deps=args.require_deps) rec = OracleReceipt.create( oracle="lineage", @@ -266,6 +300,88 @@ def _dispatch(args: argparse.Namespace) -> int: ) return _emit(rec, args.json) + if cmd == "repair": + if args.retrieve_only: + ret = retrieve_plane1_excerpt( + family=args.family, + prompt_text=args.prompt, + max_tokens=args.retrieval_max_tokens, + ) + rec = OracleReceipt.create( + oracle="plane1_retrieval", + passed=int(ret["estimated_tokens"]) <= 1500, + epistemic_status="VERIFIED", + checks={k: v for k, v in ret.items() if k not in ("text", "context")}, + findings=[], + ) + return _emit(rec, args.json) + if not args.dataset: + raise ValueError("--dataset is required unless --retrieve-only is passed") + ds_path = Path(args.dataset) + rows = [json.loads(ln) for ln in ds_path.read_text(encoding="utf-8").splitlines() if ln.strip()] + lora_cfg = json.loads(args.lora_json) + gen_fn = make_llama_server_generator( + args.server_url, + lora=lora_cfg, + max_tokens=args.max_predict, + ) + ckpt_p = Path(args.checkpoint_jsonl) if args.checkpoint_jsonl else None + split_rep = run_repair_split( + rows, + gen_fn, + max_turns=args.max_turns, + retrieval_max_tokens=args.retrieval_max_tokens, + include_plane1_retrieval=not args.no_plane1, + checkpoint_jsonl=ckpt_p, + ) + if args.output_json: + out_p = Path(args.output_json) + out_p.parent.mkdir(parents=True, exist_ok=True) + out_p.write_text(json.dumps(split_rep, indent=2, sort_keys=True) + "\n", encoding="utf-8", newline="\n") + rec = OracleReceipt.create( + oracle="repair_loop", + passed=True, + epistemic_status="VERIFIED", + checks={k: v for k, v in split_rep.items() if k != "items"}, + findings=[], + ) + return _emit(rec, args.json) + + if cmd == "audit-cert": + raw_cert = _read_text_or_stdin(args.cert_file) + try: + cert_obj = loads_strict_certificate_json(raw_cert) + except ValueError as exc: + rec = OracleReceipt.create( + oracle="transition_certificate", + passed=False, + epistemic_status="REJECTED", + checks={"finding_count": 1}, + findings=[{"rule": "CERT_STRICT_JSON_PARSE_REJECTED", "message": str(exc)}], + ) + return _emit(rec, args.json) + rep_c = audit_transition_certificate(cert_obj) + rec = OracleReceipt.create( + oracle="transition_certificate", + passed=bool(rep_c["passed"]), + epistemic_status=str(rep_c["epistemic_status"]), + checks={"finding_count": rep_c["finding_count"]}, + findings=rep_c["findings"], + ) + return _emit(rec, args.json) + + if cmd == "sandbox-cell": + cell_src = _read_text_or_stdin(args.code_file) + sbx = run_restricted_sandbox_cell(cell_src, timeout_sec=args.timeout) + rec = OracleReceipt.create( + oracle="sandbox_cell", + passed=bool(sbx["ok"]), + epistemic_status="VERIFIED" if sbx["ok"] else "REJECTED", + checks=sbx, + findings=[{"rule": "SANDBOX_VIOLATION", "message": v} for v in sbx.get("violations", [])], + ) + return _emit(rec, args.json) + if cmd == "selftest": rec = run_all_reference_selftests(allow_unsandboxed=args.allow_unsandboxed) return _emit(rec, args.json) diff --git a/src/mcproof/domain_protocol.py b/src/mcproof/domain_protocol.py new file mode 100644 index 0000000..e18ff2e --- /dev/null +++ b/src/mcproof/domain_protocol.py @@ -0,0 +1,1071 @@ +"""Pluggable domain-general `DomainOracle` protocol, budget ledger, and certificate auditor. + +Defines the clean domain boundary for `Proven-Commons/mcproof` (`mde:formal-verification`): +1. `DomainOracle` (`typing.Protocol`): pluggable interface implemented by domain-general mathematical + and coding verifiers (`FormalVerificationDomainOracle`) as well as external specialist MDEs in + separate repositories without introducing domain-specific game or application code into `mcproof`. +2. `BudgetLedger`: exact integer accounting for turns, tokens, transition steps, and sandbox calls. +3. `run_restricted_sandbox_cell`: 4.0-second, network-free, filesystem-isolated Python micro-computation + sandbox usable by any `DomainOracle` closed-loop repair runner. +4. `audit_transition_certificate`, `verify_dual_product_parity`, and `audit_portfolio_bounds`: + deterministic soundness checkers enforcing non-tautological `must_fail_controls` (`A6`), rejecting + `LINT_PASS` / `LOWER_BOUND` tier-swap promotions (`A7`), and catching all 8 abstract certificate + unsoundness classes (`F-01` through `F-08`). +""" + +from __future__ import annotations + +import ast +from collections import deque +from dataclasses import dataclass +from fractions import Fraction +import json +import os +from pathlib import Path +import re +import signal +import subprocess +import sys +import tempfile +from typing import Any, Callable, Mapping, Protocol, Sequence, runtime_checkable + +from .repair_loop import ( + MAX_RETRIEVAL_TOKENS_HARD_CAP, + default_inference_oracle, + format_diagnostics, + retrieve_plane1_excerpt, + sanitize_oracle_diagnostics, +) + +MAX_SANDBOX_TIMEOUT_SEC = 4.0 +MAX_CERT_ABS_INT = 10**60 +STRICT_RATIONAL_RE = re.compile(r"^-?[0-9]+(?:/[1-9][0-9]*)?$") +SHA256_64_RE = re.compile(r"^[0-9a-f]{64}$") +NON_VERIFIED_SUBTIERS: frozenset[str] = frozenset( + { + "LINT_PASS", + "LOWER_BOUND", + "LOWER_BOUND_UNWITNESSED", + "VERIFIED_ON_BOUNDED_REACHABLE_SUBGRAPH", + "CONJECTURE", + "UNVERIFIED_CALLER_BOUNDS", + } +) + +ALLOWLISTED_SANDBOX_MODULES: frozenset[str] = frozenset( + { + "__future__", + "bisect", + "collections", + "copy", + "dataclasses", + "decimal", + "enum", + "fractions", + "functools", + "hashlib", + "heapq", + "itertools", + "json", + "math", + "numbers", + "operator", + "re", + "statistics", + "string", + "typing", + } +) + +FORBIDDEN_SANDBOX_BUILTINS: frozenset[str] = frozenset( + { + "__import__", + "breakpoint", + "compile", + "delattr", + "eval", + "exec", + "getattr", + "globals", + "input", + "locals", + "open", + "setattr", + "vars", + } +) + +FORBIDDEN_SANDBOX_DUNDERS: frozenset[str] = frozenset( + { + "__bases__", + "__builtins__", + "__code__", + "__dict__", + "__globals__", + "__loader__", + "__mro__", + "__spec__", + "__subclasses__", + "ag_frame", + "cr_frame", + "f_back", + "f_builtins", + "f_code", + "f_globals", + "f_locals", + "gi_frame", + "tb_frame", + } +) + +FORBIDDEN_SANDBOX_MODULE_HOPS: frozenset[str] = frozenset( + { + "_builtins", + "_frozen_importlib", + "_imp", + "_io", + "_os", + "_posixsubprocess", + "_sys", + "builtins", + "codecs", + "importlib", + "io", + "modules", + "nt", + "open", + "os", + "pathlib", + "posix", + "shutil", + "socket", + "subprocess", + "sys", + } +) + + +@dataclass(frozen=True) +class DomainDiagnostic: + """Sanitized, label-free diagnostic emitted by a deterministic Tier 2 `DomainOracle`.""" + + rule: str + message: str + line: int | None = None + + def to_dict(self) -> dict[str, Any]: + out: dict[str, Any] = {"rule": self.rule, "message": self.message} + if self.line is not None: + out["line"] = self.line + return out + + +@dataclass(frozen=True) +class DomainGradeResult: + """Deterministic Tier 2 verification verdict for a candidate completion or witness.""" + + passed: bool + epistemic_status: str + diagnostics: tuple[DomainDiagnostic, ...] + metrics: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + return { + "passed": self.passed, + "epistemic_status": self.epistemic_status, + "diagnostics": [d.to_dict() for d in self.diagnostics], + "metrics": dict(self.metrics), + } + + +@dataclass +class BudgetLedger: + """Exact integer budget accounting for closed-loop formal synthesis and state-space exploration.""" + + max_turns: int = 4 + max_tokens: int = 16384 + max_steps: int = 10000 + max_sandbox_calls: int = 4 + turns_used: int = 0 + tokens_used: int = 0 + steps_used: int = 0 + sandbox_calls_used: int = 0 + format_errors: int = 0 + + def exhausted(self) -> bool: + return ( + self.turns_used >= self.max_turns + or self.tokens_used >= self.max_tokens + or self.steps_used >= self.max_steps + ) + + def utilization_ratio(self) -> Fraction: + if self.max_turns <= 0: + return Fraction(1, 1) + return Fraction(self.turns_used, self.max_turns) + + def to_dict(self) -> dict[str, Any]: + return { + "max_turns": self.max_turns, + "max_tokens": self.max_tokens, + "max_steps": self.max_steps, + "max_sandbox_calls": self.max_sandbox_calls, + "turns_used": self.turns_used, + "tokens_used": self.tokens_used, + "steps_used": self.steps_used, + "sandbox_calls_used": self.sandbox_calls_used, + "format_errors": self.format_errors, + "budget_exhausted": self.exhausted(), + } + + +@runtime_checkable +class DomainOracle(Protocol): + """Pluggable domain-general interface for Plane 1 retrieval and Tier 2 closed-loop repair.""" + + @property + def domain_id(self) -> str: ... + + def retrieve_plane1_context( + self, + task: Mapping[str, Any], + prior_diagnostics: Sequence[DomainDiagnostic] = (), + max_tokens: int = 1250, + ) -> str: ... + + def grade_candidate( + self, + task: Mapping[str, Any], + candidate: str, + ) -> DomainGradeResult: ... + + def format_repair_feedback( + self, + task: Mapping[str, Any], + grade: DomainGradeResult, + ) -> str: ... + + +class FormalVerificationDomainOracle: + """Label-free inference-time `DomainOracle` for `mcproof` Families A, B, C, and D.""" + + def __init__(self, *, spec_path: Path | None = None) -> None: + self._spec_path = spec_path + + @property + def domain_id(self) -> str: + return "mde:formal-verification" + + def retrieve_plane1_context( + self, + task: Mapping[str, Any], + prior_diagnostics: Sequence[DomainDiagnostic] = (), + max_tokens: int = 1250, + ) -> str: + capped = min(max(250, int(max_tokens)), MAX_RETRIEVAL_TOKENS_HARD_CAP) + fam = str(task.get("family", "A")).strip().upper() + prompt_text = str(task.get("prompt", "")) + findings_dicts = [d.to_dict() for d in prior_diagnostics] + res = retrieve_plane1_excerpt( + family=fam, + prompt_text=prompt_text, + prior_findings=findings_dicts, + max_tokens=capped, + spec_path=self._spec_path, + ) + return str(res["excerpt"]) + + def grade_candidate( + self, + task: Mapping[str, Any], + candidate: str, + ) -> DomainGradeResult: + fam = str(task.get("family", "A")).strip().upper() + public_view = {"id": task.get("id", ""), "family": fam, "prompt": task.get("prompt", "")} + raw = default_inference_oracle(public_view, candidate) + sanitized = sanitize_oracle_diagnostics(fam, raw.get("findings", ())) + diags = tuple( + DomainDiagnostic( + rule=d["rule"], + message=d["diagnostic"], + line=int(d["line"]) if "line" in d else None, + ) + for d in sanitized + ) + return DomainGradeResult( + passed=bool(raw["passed"]), + epistemic_status=str(raw["epistemic_status"]), + diagnostics=diags, + metrics={"raw_finding_count": len(raw.get("findings", ()))}, + ) + + def format_repair_feedback( + self, + task: Mapping[str, Any], + grade: DomainGradeResult, + ) -> str: + fam = str(task.get("family", "A")).strip().upper() + return format_diagnostics( + [d.to_dict() for d in grade.diagnostics], + family=fam, + ) + + +def audit_sandbox_cell_ast( + code: str, + *, + extra_forbidden_modules: Sequence[str] = (), + strict_exact_math: bool = False, +) -> list[str]: + """Static AST check enforcing pure-stdlib module allowlist, zero I/O, and optional exact-math checks.""" + try: + tree = ast.parse(code) + except SyntaxError as exc: + return [f"SyntaxError: line {exc.lineno}: {exc.msg}"] + + extra_forb = {str(m).split(".")[0] for m in extra_forbidden_modules} + allowed_mods = ALLOWLISTED_SANDBOX_MODULES - extra_forb + violations: list[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + top = alias.name.split(".")[0] + if top not in allowed_mods: + violations.append(f"forbidden_import:{top}") + elif isinstance(node, ast.ImportFrom): + top = (node.module or "").split(".")[0] + if (node.level or 0) > 0 or top not in allowed_mods: + violations.append(f"forbidden_from_import:{node.module or '.'}") + else: + for alias in node.names: + imp_name = alias.name + if imp_name == "*" or imp_name.startswith("_") or imp_name in FORBIDDEN_SANDBOX_MODULE_HOPS: + violations.append(f"forbidden_from_import_symbol:{imp_name}") + elif isinstance(node, ast.Attribute): + if node.attr in FORBIDDEN_SANDBOX_DUNDERS or node.attr == "__class__" or ( + node.attr.startswith("__") and node.attr.endswith("__") + ): + violations.append(f"forbidden_dunder_attr:{node.attr}") + elif node.attr in FORBIDDEN_SANDBOX_MODULE_HOPS or node.attr in ( + "system", + "popen", + "urlopen", + "urlretrieve", + "write_text", + "write_bytes", + ): + violations.append(f"forbidden_attr_call:{node.attr}") + elif isinstance(node, ast.Name) and node.id in FORBIDDEN_SANDBOX_BUILTINS: + violations.append(f"forbidden_builtin:{node.id}") + elif isinstance(node, ast.Call): + if isinstance(node.func, ast.Name): + if node.func.id in FORBIDDEN_SANDBOX_BUILTINS: + violations.append(f"forbidden_builtin:{node.func.id}") + elif strict_exact_math and node.func.id == "float": + violations.append("forbidden_float_coercion:float") + elif strict_exact_math and node.func.id == "pow" and len(node.args) == 2: + exp_arg = node.args[1] + if (isinstance(exp_arg, ast.Constant) and type(exp_arg.value) is int and exp_arg.value < 0) or ( + isinstance(exp_arg, ast.UnaryOp) and isinstance(exp_arg.op, ast.USub) + ): + violations.append("forbidden_negative_pow:pow") + elif strict_exact_math and isinstance(node, ast.BinOp): + if isinstance(node.op, ast.Div): + violations.append("forbidden_float_div:/") + elif isinstance(node.op, ast.Pow): + if (isinstance(node.right, ast.Constant) and type(node.right.value) is int and node.right.value < 0) or ( + isinstance(node.right, ast.UnaryOp) and isinstance(node.right.op, ast.USub) + ): + violations.append("forbidden_negative_pow:**") + elif strict_exact_math and isinstance(node, ast.Constant) and isinstance(node.value, float): + violations.append(f"forbidden_float_literal:{node.value!r}") + return violations + + +def run_restricted_sandbox_cell( + code: str, + *, + timeout_sec: float = MAX_SANDBOX_TIMEOUT_SEC, + injected_globals: Mapping[str, int | str | bool | list[Any] | dict[str, Any]] | None = None, + extra_forbidden_modules: Sequence[str] = (), + strict_exact_math: bool = False, +) -> dict[str, Any]: + """Execute a pure-algorithmic Python micro-computation cell under a hard <= 4.0s wall-clock cap.""" + capped_timeout = min(max(0.1, float(timeout_sec)), MAX_SANDBOX_TIMEOUT_SEC) + ast_violations = audit_sandbox_cell_ast( + code, + extra_forbidden_modules=extra_forbidden_modules, + strict_exact_math=strict_exact_math, + ) + if ast_violations: + return { + "ok": False, + "returncode": 2, + "timed_out": False, + "ast_rejected": True, + "violations": ast_violations, + "timeout_sec": capped_timeout, + "stdout": "", + "stderr": "; ".join(ast_violations), + } + + extra_forb = {str(m).split(".")[0] for m in extra_forbidden_modules} + allowed_sorted = sorted(ALLOWLISTED_SANDBOX_MODULES - extra_forb) + preimport_mods = [m for m in allowed_sorted if m != "__future__"] + preamble_lines: list[str] = [] + if injected_globals: + for key in sorted(injected_globals.keys()): + if re.match(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$", key): + preamble_lines.append(f"_ns[{key!r}] = {injected_globals[key]!r}") + + with tempfile.TemporaryDirectory(prefix="mcproof_sandbox_") as tmp_dir: + if os.name != "nt": + os.chmod(tmp_dir, 0o700) + code_hex = code.encode("utf-8").hex() + driver_src = ( + "import " + ", ".join(preimport_mods) + "\n" + "import builtins as _b\n" + f"_ALLOWLIST = frozenset({allowed_sorted!r})\n" + "_orig_import = _b.__import__\n" + "_orig_compile = _b.compile\n" + "_orig_exec = _b.exec\n" + "def _guarded_import(name, globals=None, locals=None, fromlist=(), level=0):\n" + " if level and level > 0:\n" + " raise ImportError('relative imports are forbidden in sandbox')\n" + " top = str(name).split('.')[0]\n" + " if top not in _ALLOWLIST:\n" + " raise ImportError(f'module {top!r} is not in sandbox allowlist')\n" + " return _orig_import(name, globals, locals, fromlist, level)\n" + "_safe_b = dict(_b.__dict__)\n" + "_safe_b['__import__'] = _guarded_import\n" + "for _k in ('open', 'input', 'breakpoint', 'eval', 'exec', 'compile'):\n" + " _safe_b.pop(_k, None)\n" + "_ns = {'__name__': '__main__', '__builtins__': _safe_b}\n" + + ("\n".join(preamble_lines) + "\n" if preamble_lines else "") + + f"_src = bytes.fromhex({code_hex!r}).decode('utf-8')\n" + "_co = _orig_compile(_src, '', 'exec')\n" + "import sys as _sys\n" + "_sys.addaudithook(lambda ev, _a: (_ for _ in ()).throw(PermissionError(ev)) " + "if ev in ('open','os.system','os.mkdir','os.remove','os.unlink','os.rmdir','os.rename') " + "or ev.startswith(('subprocess.','socket.')) else None)\n" + "del _sys\n" + "_orig_exec(_co, _ns, _ns)\n" + ) + driver_path = Path(tmp_dir) / "cell_driver.py" + driver_path.write_text(driver_src, encoding="utf-8", newline="\n") + + clean_env: dict[str, str] = { + "PATH": os.environ.get("PATH", "") if os.name == "nt" else "/usr/bin:/bin", + "PYTHONHASHSEED": "0", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONNOUSERSITE": "1", + } + if os.name == "nt": + for win_key in ("SystemRoot", "SYSTEMROOT", "WINDIR", "TEMP", "TMP"): + if win_key in os.environ: + clean_env[win_key] = os.environ[win_key] + proc = subprocess.Popen( + [sys.executable, "-I", "-B", "-O", str(driver_path)], + cwd=tmp_dir, + env=clean_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=(os.name != "nt"), + ) + try: + out, err = proc.communicate(timeout=capped_timeout) + return { + "ok": proc.returncode == 0, + "returncode": int(proc.returncode), + "timed_out": False, + "ast_rejected": False, + "violations": [], + "timeout_sec": capped_timeout, + "stdout": out[:8000], + "stderr": err[:2000], + } + except subprocess.TimeoutExpired: + kill_err: str | None = None + try: + if hasattr(os, "killpg") and hasattr(signal, "SIGKILL"): + os.killpg(proc.pid, signal.SIGKILL) + else: + proc.kill() + except OSError as exc: + kill_err = str(exc) + out, err = proc.communicate() + return { + "ok": False, + "returncode": -9, + "timed_out": True, + "ast_rejected": False, + "violations": ["sandbox_timeout_exceeded_4s"], + "timeout_sec": capped_timeout, + "stdout": (out or "")[:4000], + "stderr": f"TimeoutExpired: cell exceeded {capped_timeout:.1f}s limit" + + (f" ({kill_err})" if kill_err else ""), + } + + +def verify_dual_product_parity( + *, + spec_init: Any, + impl_init: Any, + spec_actions: Callable[[Any], Sequence[Any]], + impl_actions: Callable[[Any], Sequence[Any]], + spec_step: Callable[[Any, Any], Any], + impl_step: Callable[[Any, Any], Any], + spec_invariant: Callable[[Any], bool], + impl_invariant: Callable[[Any], bool], + spec_is_goal: Callable[[Any], bool], + impl_is_goal: Callable[[Any], bool], + spec_obs: Callable[[Any], Any], + impl_obs: Callable[[Any], Any], + max_product_states: int = 10000, +) -> dict[str, Any]: + """Product-graph `(s_spec, u_impl)` verifier closing `F-06` coarse-observation & collapsed-spec bugs.""" + visited: set[tuple[Any, Any]] = {(spec_init, impl_init)} + queue: deque[tuple[Any, Any]] = deque([(spec_init, impl_init)]) + mismatches: list[dict[str, str]] = [] + + while queue: + s_state, u_state = queue.popleft() + if spec_obs(s_state) != impl_obs(u_state): + mismatches.append({"rule": "F06_OBSERVATION_DIVERGENCE", "message": f"obs({s_state!r}) != obs({u_state!r})"}) + break + s_inv = bool(spec_invariant(s_state)) + u_inv = bool(impl_invariant(u_state)) + if s_inv != u_inv or not u_inv: + mismatches.append( + { + "rule": "F06_IMPL_INVARIANT_VIOLATION_OR_DIVERGENCE", + "message": f"spec.invariant={s_inv}, impl.invariant={u_inv} at product state", + } + ) + break + if bool(spec_is_goal(s_state)) != bool(impl_is_goal(u_state)): + mismatches.append({"rule": "F06_GOAL_PARITY_MISMATCH", "message": "spec.is_goal != impl.is_goal"}) + break + s_acts = sorted(spec_actions(s_state), key=repr) + u_acts = sorted(impl_actions(u_state), key=repr) + if s_acts != u_acts: + mismatches.append({"rule": "F06_ACTION_SET_MISMATCH", "message": "spec.actions != impl.actions"}) + break + for act in s_acts: + nxt = (spec_step(s_state, act), impl_step(u_state, act)) + if nxt not in visited: + if len(visited) >= max_product_states: + return { + "passed": False, + "complete_space": False, + "epistemic_status": "REJECTED_INCOMPLETE_PRODUCT_SPACE", + "states_checked": len(visited), + "findings": [ + { + "rule": "F05_INCOMPLETE_PRODUCT_GRAPH", + "message": f"product BFS truncated at max_product_states={max_product_states}", + } + ], + } + visited.add(nxt) + queue.append(nxt) + + passed = len(mismatches) == 0 + return { + "passed": passed, + "complete_space": True, + "epistemic_status": "PROVEN" if passed else "REJECTED_DIFFERENTIAL_MISMATCH", + "states_checked": len(visited), + "findings": mismatches, + } + + +def audit_portfolio_bounds( + items: Sequence[Mapping[str, Any]], + *, + registered_verified_ids: frozenset[str] = frozenset(), +) -> dict[str, Any]: + """Audit portfolio lower/upper bound records, closing `F-07` caller-bound inflation and bound inversions.""" + findings: list[dict[str, str]] = [] + triaged: list[dict[str, Any]] = [] + for row in items: + iid = str(row.get("id", "unknown")) + p_floor = row.get("proven_floor") + s_known = row.get("shortest_known") + l_rec = row.get("live_record") + + if p_floor is not None and s_known is not None and int(p_floor) > int(s_known): + findings.append( + { + "rule": "CERT_F07_INCONSISTENT_PORTFOLIO_BOUNDS", + "message": f"{iid}: proven_floor ({p_floor}) > shortest_known ({s_known})", + } + ) + status = "REJECTED_INCONSISTENT_PORTFOLIO_BOUNDS" + elif s_known is not None and l_rec is not None and int(s_known) > int(l_rec): + findings.append( + { + "rule": "CERT_F07_INCONSISTENT_PORTFOLIO_BOUNDS", + "message": f"{iid}: shortest_known ({s_known}) > live_record ({l_rec})", + } + ) + status = "REJECTED_INCONSISTENT_PORTFOLIO_BOUNDS" + elif iid not in registered_verified_ids: + if str(row.get("epistemic_status", "")).upper() in ("VERIFIED", "PROVEN"): + findings.append( + { + "rule": "CERT_F07_UNVERIFIED_CALLER_BOUNDS", + "message": f"{iid}: caller-supplied bounds lack registered proof/witness certificates", + } + ) + status = "UNVERIFIED_CALLER_BOUNDS" + else: + status = "VERIFIED" + triaged.append({"id": iid, "epistemic_status": status, "caller_bounds_verified": iid in registered_verified_ids}) + + passed = len(findings) == 0 + return { + "passed": passed, + "epistemic_status": "VERIFIED" if passed else "REJECTED", + "items": triaged, + "findings": findings, + } + + +def audit_transition_certificate( + cert: Mapping[str, Any], + *, + require_must_fail_controls: bool = False, + registered_artifact_digests: frozenset[str] | None = None, +) -> dict[str, Any]: + """Domain-general fail-closed auditor covering `F-01..F-08` plus `A6`/`A7` control and tier-swap gates.""" + findings: list[dict[str, str]] = [] + status = str(cert.get("epistemic_status", "")).strip().upper() + proof_mode = str(cert.get("proof_mode", "")).strip().lower() + prior_status = str(cert.get("prior_status", "")).strip().upper() + evidence_kind = str(cert.get("evidence_kind", "")).strip().lower() + + # A7: Reject tier-swap promotions of LINT_PASS / LOWER_BOUND / bounded-subgraph to VERIFIED or PROVEN + if status in ("VERIFIED", "PROVEN"): + if prior_status in NON_VERIFIED_SUBTIERS or evidence_kind in ("static_lint", "lower_bound_only", "bounded_subgraph"): + findings.append( + { + "rule": "CERT_A7_ILLEGAL_TIER_SWAP", + "message": f"Cannot promote {prior_status or evidence_kind} to {status} without full replayed witness/proof.", + } + ) + if cert.get("valid_winning_witness") is False or cert.get("valid_winning_route") is False: + findings.append( + { + "rule": "CERT_A7_ILLEGAL_TIER_SWAP", + "message": f"Rejected witness cannot carry epistemic_status={status}.", + } + ) + + # A6: Non-empty, distinct, non-tautological must_fail_controls + if require_must_fail_controls or "must_fail_controls" in cert or "control_records" in cert: + controls = cert.get("must_fail_controls") + if not isinstance(controls, list) or len(controls) == 0: + findings.append( + { + "rule": "CERT_A6_MISSING_OR_DUPLICATE_CONTROLS", + "message": "Certificate must include a non-empty must_fail_controls list.", + } + ) + else: + clean_names = [str(c).strip() for c in controls if str(c).strip()] + if len(clean_names) != len(controls) or len(set(clean_names)) != len(clean_names): + findings.append( + { + "rule": "CERT_A6_MISSING_OR_DUPLICATE_CONTROLS", + "message": "must_fail_controls contains empty or duplicate control identifiers.", + } + ) + for crec in cert.get("control_records") or []: + if not isinstance(crec, Mapping): + continue + cname = str(crec.get("name", "unnamed_control")) + if crec.get("unexpectedly_passed") is True: + findings.append( + { + "rule": "CERT_A6_CONTROL_PASSED", + "message": f"Negative control {cname!r} unexpectedly passed.", + } + ) + if crec.get("oracle_invoked") is False or crec.get("seed_equals_target") is True: + findings.append( + { + "rule": "CERT_A6_TAUTOLOGICAL_CONTROL", + "message": f"Negative control {cname!r} is tautological (zero oracle invocation or pre-seeded at target).", + } + ) + + # F-01: Optimistic differential boolean without >=2 independent replay engines + oracles = cert.get("oracles_checked") or cert.get("engines_verified") or cert.get("checkers_executed") or [] + if cert.get("differential_match") is True and (not isinstance(oracles, list) or len(oracles) < 2): + findings.append( + { + "rule": "CERT_SINGLE_ORACLE_DIFFERENTIAL", + "message": "differential_match=True requires >=2 independent verification engines.", + } + ) + + # F-02: Partial-projection state hash missing hidden state bits + if cert.get("includes_hidden_state") is False or cert.get("state_hash_mode") in ("observable_projection_only", "ascii_only", "rendered_board_only"): + findings.append( + { + "rule": "CERT_PARTIAL_STATE_PROJECTION", + "message": "State digest omits hidden transition state; full canonical state commitment required.", + } + ) + + # F-03: Unpinned TCB manifest or missing 64-hex checker/auditor SHA-256 (fail closed when omitted, NEON-D5) + pins = cert.get("tcb_pins") + if status in ("VERIFIED", "PROVEN", "LOWER_BOUND"): + if not isinstance(pins, Mapping): + findings.append( + { + "rule": "CERT_TCB_MANIFEST_UNLOCKED", + "message": "Certified record omits mandatory tcb_pins manifest commitment.", + } + ) + else: + chk_sha = str(pins.get("checker_sha256") or pins.get("checker_auditor_py_sha256") or "") + if not pins.get("tcb_manifest_verified", False) or not SHA256_64_RE.match(chk_sha): + findings.append( + { + "rule": "CERT_TCB_MANIFEST_UNLOCKED", + "message": "TCB pins must verify SHA256SUMS and bind a 64-hex checker/auditor SHA-256.", + } + ) + + # NEON-D2: Truncated search budget cannot claim exact optimum or PROVEN status + if cert.get("truncated_by_budget") is True and ( + status == "PROVEN" + or str(cert.get("certificate_kind", "")).strip().upper() == "EXACT_DIJKSTRA_OPTIMUM" + or cert.get("exact_optimum_reached") is True + ): + findings.append( + { + "rule": "CERT_TRUNCATED_SEARCH_BUDGET", + "message": "Search truncated by budget cannot claim PROVEN or EXACT_DIJKSTRA_OPTIMUM.", + } + ) + + # F-04: Unwitnessed or mismatched-length `proven_floor` claimed as `VERIFIED` + if "proven_floor" in cert: + p_floor = int(cert["proven_floor"]) + max_b = cert.get("max_bound") + if max_b is not None and p_floor > int(max_b): + findings.append( + { + "rule": "CERT_FLOOR_EXCEEDS_MAX_BOUND", + "message": f"proven_floor ({p_floor}) exceeds search max_bound ({max_b}).", + } + ) + if status == "VERIFIED": + witness = cert.get("witness") if "witness" in cert else cert.get("winning_witness") + witness_cost = cert.get("witness_cost") + if witness_cost is None and isinstance(witness, str): + witness_cost = len(witness) + if cert.get("exact_optimum_reached") is False or not witness: + findings.append( + { + "rule": "CERT_UNWITNESSED_VERIFIED_OPTIMUM", + "message": "Unwitnessed lower bound must carry status LOWER_BOUND_UNWITNESSED, never VERIFIED.", + } + ) + elif witness_cost is not None and int(witness_cost) != p_floor: + findings.append( + { + "rule": "CERT_WITNESS_FLOOR_MISMATCH", + "message": f"Witness cost {witness_cost} != claimed proven_floor {p_floor}.", + } + ) + + # Instance mismatch & step-consistency violations + if cert.get("instance_sha256") and cert.get("canonical_instance_sha256"): + if cert.get("instance_sha256") != cert.get("canonical_instance_sha256"): + findings.append( + { + "rule": "CERT_INSTANCE_DIGEST_MISMATCH", + "message": "Certificate instance_sha256 does not match canonical target instance.", + } + ) + if int(cert.get("step_consistency_violations", 0)) > 0: + findings.append( + { + "rule": "CERT_STEP_INCONSISTENCY", + "message": "Layered search lower bound is invalid when step_consistency_violations > 0.", + } + ) + + # F-05: Truncated inductive potential on incomplete concrete state graph + incomplete_graph = ( + cert.get("complete_space") is False + or cert.get("complete_domain_closed") is False + or int(cert.get("open_transitions", 0)) > 0 + ) + if ( + proof_mode == "inductive_potential" + or status == "VERIFIED_ON_BOUNDED_REACHABLE_SUBGRAPH" + or (cert.get("certified_lower_bound") is not None and incomplete_graph) + ): + if incomplete_graph or cert.get("complete_space") is not True and cert.get("complete_domain_closed") is not True: + findings.append( + { + "rule": "CERT_TRUNCATED_INDUCTIVE_POTENTIAL", + "message": "inductive_potential or certified_lower_bound on an incomplete concrete graph is unsound.", + } + ) + + # F-06: Spec-only PROVEN under coarse project_obs or skipped impl parity + if status == "PROVEN" and ( + cert.get("impl_invariant_checked") is False + or cert.get("impl_goal_parity_checked") is False + or cert.get("obs_projection_injective") is False + ): + findings.append( + { + "rule": "CERT_IMPL_INVARIANT_SKIPPED", + "message": "PROVEN status is unsound when implementation invariant/goal parity is unchecked under coarse observation.", + } + ) + + # F-07: Unverified caller bounds stamped VERIFIED + if (cert.get("unverified_caller_claim") is True or cert.get("caller_bounds_verified") is False) and status in ("VERIFIED", "PROVEN"): + findings.append( + { + "rule": "CERT_UNVERIFIED_CALLER_INFLATION", + "message": "Caller-supplied numeric bounds without registered certificate verification cannot be stamped VERIFIED.", + } + ) + + # F-08: Missing, malformed non-64-hex, or unregistered witness_or_proof_sha256 (fail closed, NEON-D5) + if status in ("VERIFIED", "PROVEN"): + w_sha = str(cert.get("witness_or_proof_sha256", "")) + if not SHA256_64_RE.match(w_sha): + findings.append( + { + "rule": "CERT_INVALID_WITNESS_SHA256", + "message": f"witness_or_proof_sha256 must be a lowercase 64-hex SHA-256 digest (got {w_sha!r}).", + } + ) + elif registered_artifact_digests is not None and w_sha not in registered_artifact_digests: + findings.append( + { + "rule": "CERT_UNREGISTERED_WITNESS_DIGEST", + "message": "witness_or_proof_sha256 is absent from verified certificate registry.", + } + ) + + passed = len(findings) == 0 + return { + "passed": passed, + "epistemic_status": "VERIFIED" if passed else "REJECTED", + "finding_count": len(findings), + "findings": findings, + } + + +def loads_strict_certificate_json(raw_json: str, *, max_abs_int: int = MAX_CERT_ABS_INT) -> Any: + """Parse certificate JSON rejecting duplicate keys, floats, NaN/Infinity, and oversized integers.""" + + def _reject_float(token: str) -> None: + raise ValueError(f"floating-point JSON literal forbidden in exact certificate: {token!r}") + + def _reject_const(token: str) -> None: + raise ValueError(f"non-finite JSON constant forbidden in exact certificate: {token!r}") + + def _bounded_int(token: str) -> int: + if len(token.lstrip("-")) > 62: + raise ValueError(f"integer literal exceeds safe digit bound ({len(token)} chars)") + val = int(token) + if abs(val) > max_abs_int: + raise ValueError(f"integer literal exceeds |v| <= {max_abs_int} cap") + return val + + def _no_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + out: dict[str, Any] = {} + for k, v in pairs: + if k in out: + raise ValueError(f"duplicate JSON object key forbidden: {k!r}") + out[k] = v + return out + + return json.loads( + raw_json, + object_pairs_hook=_no_duplicate_keys, + parse_int=_bounded_int, + parse_float=_reject_float, + parse_constant=_reject_const, + ) + + +def _parse_exact_rational(val: Any, *, max_abs_int: int = MAX_CERT_ABS_INT) -> Fraction: + """Parse an exact `int` (`type(val) is int`, never `bool`) or canonical `'p/q'` string into `Fraction`.""" + if type(val) is int: + if abs(val) > max_abs_int: + raise ValueError("integer exceeds cap") + return Fraction(val, 1) + if isinstance(val, str) and STRICT_RATIONAL_RE.match(val): + if "/" in val: + n_str, d_str = val.split("/", 1) + num, den = int(n_str), int(d_str) + else: + num, den = int(val), 1 + if abs(num) > max_abs_int or abs(den) > max_abs_int: + raise ValueError("rational component exceeds cap") + return Fraction(num, den) + raise ValueError(f"invalid exact rational token {val!r} (bool/float forbidden)") + + +def verify_rational_lp_certificate(cert: Mapping[str, Any]) -> dict[str, Any]: + """Verify exact rational LP dual lower-bound (`bounds`) or Farkas (`infeasible`) certificates.""" + mode = str(cert.get("mode", "bounds")).strip().lower() + findings: list[dict[str, str]] = [] + try: + matrix_a = [[_parse_exact_rational(x) for x in row] for row in cert["A"]] + rhs_b = [_parse_exact_rational(x) for x in cert["b"]] + dual_y = [_parse_exact_rational(x) for x in cert["y"]] + except (KeyError, TypeError, ValueError) as exc: + return { + "passed": False, + "epistemic_status": "REJECTED", + "findings": [{"rule": "LP_CERT_MALFORMED", "message": str(exc)}], + } + + m_rows = len(matrix_a) + n_cols = len(matrix_a[0]) if m_rows > 0 else 0 + if len(rhs_b) != m_rows or len(dual_y) != m_rows or any(len(r) != n_cols for r in matrix_a): + return { + "passed": False, + "epistemic_status": "REJECTED", + "findings": [{"rule": "LP_CERT_DIMENSION_MISMATCH", "message": "A, b, y dimensions do not align"}], + } + + if any(yv < 0 for yv in dual_y): + findings.append({"rule": "LP_DUAL_NEGATIVE_MULTIPLIER", "message": "dual vector y must satisfy y >= 0"}) + + # Compute y^T A (length n_cols) and y^T b (scalar) for primal system A x >= b, x >= 0 + yt_a = [sum((dual_y[i] * matrix_a[i][j] for i in range(m_rows)), Fraction(0, 1)) for j in range(n_cols)] + yt_b = sum((dual_y[i] * rhs_b[i] for i in range(m_rows)), Fraction(0, 1)) + + if mode == "bounds": + try: + cost_c = [_parse_exact_rational(x) for x in cert["c"]] + claimed_lb = _parse_exact_rational(cert["claimed_lower_bound"]) + except (KeyError, TypeError, ValueError) as exc: + return { + "passed": False, + "epistemic_status": "REJECTED", + "findings": [{"rule": "LP_CERT_MALFORMED", "message": str(exc)}], + } + if len(cost_c) != n_cols: + findings.append({"rule": "LP_CERT_DIMENSION_MISMATCH", "message": "len(c) != cols(A)"}) + else: + for j in range(n_cols): + if yt_a[j] > cost_c[j]: + findings.append( + { + "rule": "LP_DUAL_INFEASIBLE_COLUMN", + "message": f"column {j}: (y^T A)[{j}] = {yt_a[j]} exceeds c[{j}] = {cost_c[j]}", + } + ) + break + if yt_b < claimed_lb: + findings.append( + { + "rule": "LP_DUAL_OBJECTIVE_BELOW_CLAIM", + "message": f"y^T b = {yt_b} < claimed_lower_bound = {claimed_lb}", + } + ) + elif mode == "infeasible": + # Farkas certificate for {A x >= b, x >= 0} = empty: y >= 0, y^T A <= 0, and y^T b > 0 + for j in range(n_cols): + if yt_a[j] > 0: + findings.append( + { + "rule": "LP_FARKAS_POSITIVE_COLUMN", + "message": f"Farkas certificate requires (y^T A)[{j}] <= 0, got {yt_a[j]}", + } + ) + break + if yt_b <= 0: + findings.append( + { + "rule": "LP_FARKAS_NONPOSITIVE_WITNESS", + "message": f"Farkas certificate requires y^T b > 0, got {yt_b}", + } + ) + else: + findings.append({"rule": "LP_CERT_UNSUPPORTED_MODE", "message": f"unsupported LP mode {mode!r}"}) + + passed = len(findings) == 0 + return { + "passed": passed, + "epistemic_status": "VERIFIED" if passed else "REJECTED", + "dual_objective": f"{yt_b.numerator}/{yt_b.denominator}", + "findings": findings, + } + + +def verify_minimal_goal_stopping( + *, + init_state: Any, + actions: Sequence[Any], + step_fn: Callable[[Any, Any], Any], + is_goal_fn: Callable[[Any], bool], +) -> dict[str, Any]: + """Verify an action sequence reaches a goal state and halts without post-goal extra actions.""" + cur = init_state + if bool(is_goal_fn(cur)): + if len(actions) > 0: + return { + "passed": False, + "epistemic_status": "REJECTED", + "first_goal_step": 0, + "total_actions": len(actions), + "findings": [ + { + "rule": "CERT_POST_GOAL_EXTRA_ACTIONS", + "message": f"initial state already satisfies goal, yet {len(actions)} extra actions were emitted", + } + ], + } + return {"passed": True, "epistemic_status": "VERIFIED", "first_goal_step": 0, "total_actions": 0, "findings": []} + + for idx, act in enumerate(actions, start=1): + nxt = step_fn(cur, act) + if nxt is None: + return { + "passed": False, + "epistemic_status": "REJECTED", + "first_goal_step": None, + "total_actions": len(actions), + "findings": [{"rule": "CERT_INVALID_TRANSITION_STEP", "message": f"invalid step at index {idx}"}], + } + cur = nxt + if bool(is_goal_fn(cur)) and idx < len(actions): + return { + "passed": False, + "epistemic_status": "REJECTED", + "first_goal_step": idx, + "total_actions": len(actions), + "findings": [ + { + "rule": "CERT_POST_GOAL_EXTRA_ACTIONS", + "message": f"goal reached at step {idx}, followed by {len(actions) - idx} trailing extra actions", + } + ], + } + + reached = bool(is_goal_fn(cur)) + return { + "passed": reached, + "epistemic_status": "VERIFIED" if reached else "REJECTED", + "first_goal_step": len(actions) if reached else None, + "total_actions": len(actions), + "findings": [] if reached else [{"rule": "CERT_GOAL_NOT_REACHED", "message": "terminal state is not a goal"}], + } diff --git a/src/mcproof/lean_harness.py b/src/mcproof/lean_harness.py index e917c35..bac8b12 100644 --- a/src/mcproof/lean_harness.py +++ b/src/mcproof/lean_harness.py @@ -188,6 +188,92 @@ def strip_lean_comments_and_strings(src: str) -> str: return "".join(out) +def _unwrap_outer_parens(expr: str) -> str: + s = expr.strip() + while len(s) >= 2 and s[0] == "(" and s[-1] == ")": + depth = 0 + wraps_all = True + for idx, ch in enumerate(s): + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0 and idx < len(s) - 1: + wraps_all = False + break + if wraps_all and depth == 0: + s = s[1:-1].strip() + else: + break + return s + + +def _split_top_level_arrows(expr: str) -> list[str]: + parts: list[str] = [] + depth = 0 + start = 0 + i = 0 + n = len(expr) + while i < n: + ch = expr[i] + if ch in "([{⟨": + depth += 1 + i += 1 + elif ch in ")]}⟩": + depth = max(0, depth - 1) + i += 1 + elif depth == 0 and expr.startswith("->", i): + parts.append(expr[start:i].strip()) + i += 2 + start = i + elif depth == 0 and ch == "→": + parts.append(expr[start:i].strip()) + i += 1 + start = i + else: + i += 1 + parts.append(expr[start:].strip()) + return [p for p in parts if p] + + +def _extract_binder_props(prefix: str) -> list[tuple[str, str]]: + """Extract `(lhs_names, rhs_type)` from top-level `(...)` binders in a theorem header.""" + out: list[tuple[str, str]] = [] + i = 0 + n = len(prefix) + while i < n: + if prefix[i] == "(": + depth = 1 + j = i + 1 + colon_at = -1 + while j < n and depth > 0: + if prefix[j] in "([{⟨": + depth += 1 + elif prefix[j] in ")]}⟩": + depth -= 1 + elif prefix[j] == ":" and depth == 1 and colon_at < 0 and prefix[j : j + 2] != ":=": + colon_at = j + j += 1 + if depth == 0 and colon_at > i: + lhs = prefix[i + 1 : colon_at].strip() + rhs = prefix[colon_at + 1 : j - 1].strip() + out.append((lhs, rhs)) + i = j + else: + i += 1 + return out + + +def _is_vacuous_false_prop(raw_prop: str) -> bool: + u = re.sub(r"\s+", " ", _unwrap_outer_parens(raw_prop)).strip() + if u == "False": + return True + segs = _split_top_level_arrows(u) + if len(segs) == 2 and _unwrap_outer_parens(segs[0]) == "True" and _unwrap_outer_parens(segs[1]) == "False": + return True + return False + + def audit_weak_statements(source: str) -> list[dict[str, Any]]: """Detect WeakStatement vacuous-hypothesis and tautological-conclusion hazards.""" cleaned = strip_lean_comments_and_strings(source) @@ -199,28 +285,52 @@ def audit_weak_statements(source: str) -> list[dict[str, Any]]: line_no = cleaned.count("\n", 0, match.start()) + 1 compact = re.sub(r"\s+", " ", header).strip() - # 1. Conclusion is literal `True` - if re.search(r":\s*True\s*$", compact): + depth = 0 + top_colon_idx = -1 + for idx_c, ch in enumerate(compact): + if ch in "([{⟨": + depth += 1 + elif ch in ")]}⟩": + depth = max(0, depth - 1) + elif ch == ":" and depth == 0 and compact[idx_c : idx_c + 2] != ":=": + top_colon_idx = idx_c + break + + binder_prefix = compact[:top_colon_idx] if top_colon_idx >= 0 else compact + concl_expr = compact[top_colon_idx + 1 :].strip() if top_colon_idx >= 0 else "" + arrow_segs = _split_top_level_arrows(concl_expr) if concl_expr else [] + final_concl = _unwrap_outer_parens(arrow_segs[-1]) if arrow_segs else "" + arrow_premises = arrow_segs[:-1] if len(arrow_segs) > 1 else [] + binders = _extract_binder_props(binder_prefix) + + # 1. Conclusion is literal `True` (including parenthesized `(True)` or `P -> (True)`) + if final_concl == "True": findings.append({ "line": line_no, "rule": "VACUOUS_TRUE_CONCLUSION", "message": f"theorem '{thm_name}' proves literal True instead of a substantive property", }) - # 2. Explicit False hypothesis (`(h : False)` or `False ->` / `False →`) - if re.search(r":\s*False\b|\bFalse\s*(?:->|→)", compact): - findings.append({ - "line": line_no, - "rule": "VACUOUS_FALSE_HYPOTHESIS", - "message": f"theorem '{thm_name}' assumes False, making the implication vacuously true", - }) + # 2. Explicit False or `True -> False` hypothesis in binders or arrow premises (`(False) →`, `(h : True → False)`) + has_false_hyp = any(_is_vacuous_false_prop(rhs) for _, rhs in binders) or any( + _is_vacuous_false_prop(prem) for prem in arrow_premises + ) + if has_false_hyp or re.search(r":\s*False\b|\bFalse\s*(?:->|→)", compact): + # Exclude legitimate negation conclusion `P → False` where `False` is only the final conclusion! + if has_false_hyp or any("False" in rhs for _, rhs in binders) or any("False" in p for p in arrow_premises): + findings.append({ + "line": line_no, + "rule": "VACUOUS_FALSE_HYPOTHESIS", + "message": f"theorem '{thm_name}' assumes False (or True -> False), making the implication vacuously true", + }) # 3. Impossible Nat / self-contradictory inequalities (`n < 0` on Nat, `x < x`, `x != x`) nat_vars = set() - for binder in re.finditer(r"\(([^():]+):\s*Nat\)", compact): - for tok in binder.group(1).split(): - if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_']*", tok): - nat_vars.add(tok) + for lhs, rhs in binders: + if _unwrap_outer_parens(rhs) == "Nat": + for tok in lhs.split(): + if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_']*", tok): + nat_vars.add(tok) for v in sorted(nat_vars): if re.search(rf"\b{re.escape(v)}\s*<\s*0\b", compact): findings.append({ @@ -232,10 +342,8 @@ def audit_weak_statements(source: str) -> list[dict[str, Any]]: for self_lt in re.finditer(r"\b([A-Za-z_][A-Za-z0-9_']*)\s*(?:<|≠|!=)\s*\1\b", compact): prefix = compact[: self_lt.start()].rstrip() suffix = compact[self_lt.end() :].lstrip() - # Strip enclosing open parentheses on prefix to detect `¬(a ≠ a)` or `Not (a ≠ a)` unwrapped_prefix = prefix.rstrip("(").rstrip() negated_by_op = unwrapped_prefix.endswith("¬") or bool(re.search(r"\bNot$", unwrapped_prefix)) - # Also permit explicit irreflexive implication conclusion `: (a < a) -> False` unwrapped_suffix = suffix.lstrip(")").lstrip() negated_by_false_arrow = bool( re.match(r"^(?:->|→)\s*False\s*$", unwrapped_suffix) @@ -252,14 +360,18 @@ def audit_weak_statements(source: str) -> list[dict[str, Any]]: # 4. Direct contradictory propositional binders `(h1 : P)` and `(h2 : ¬ P)` or `(h2 : P -> False)` props: list[str] = [] neg_props: list[str] = [] - for hyp in re.finditer(r"\([^():]+:\s*([^()]+)\)", compact): - prop = hyp.group(1).strip() + for _, rhs in binders: + prop = _unwrap_outer_parens(rhs) if prop.startswith("¬"): - neg_props.append(prop[1:].strip()) + neg_props.append(_unwrap_outer_parens(prop[1:])) elif prop.startswith("Not "): - neg_props.append(prop[4:].strip()) + neg_props.append(_unwrap_outer_parens(prop[4:])) else: - props.append(prop) + segs_p = _split_top_level_arrows(prop) + if len(segs_p) == 2 and _unwrap_outer_parens(segs_p[1]) == "False": + neg_props.append(_unwrap_outer_parens(segs_p[0])) + else: + props.append(prop) overlap = set(props) & set(neg_props) if overlap: findings.append({ @@ -268,9 +380,484 @@ def audit_weak_statements(source: str) -> list[dict[str, Any]]: "message": f"theorem '{thm_name}' assumes both P and Not P ({sorted(overlap)[0]})", }) + # 5. Circular / tautological binder premise identical to conclusion (`(h : P) : P`) + if final_concl and final_concl != "True": + norm_concl = re.sub(r"\s+", " ", final_concl).strip() + for p_hyp in [*props, *[_unwrap_outer_parens(ap) for ap in arrow_premises]]: + if re.sub(r"\s+", " ", p_hyp).strip() == norm_concl: + findings.append({ + "line": line_no, + "rule": "VACUOUS_TAUTOLOGICAL_HYPOTHESIS", + "message": f"theorem '{thm_name}' assumes its exact conclusion '{norm_concl}' as a premise", + }) + break + return findings +LEAN_DEF_OR_ABBREV_RE = re.compile( + rf"(?ms)^[ \t]*{MODIFIERS_PAT}(?Pdef|abbrev)[ \t]+(?P[A-Za-z_][A-Za-z0-9_.']*)\b(?P.*?)" + r"(?=\n[ \t]*(?:@\[[^\]]*\][ \t]*)?(?:(?:scoped|local|private|protected|noncomputable|public)[ \t]+)*" + r"(?:def|abbrev|theorem|lemma|structure|inductive|class|instance|axiom|opaque|example|end|namespace)\b|\Z)" +) + +LEAN_TOKEN_RE = re.compile( + r":=|=>|<=|>=|!=|==|&&|\|\||->|<-|::|[A-Za-z_][A-Za-z0-9_.']*|[0-9]+|[()\[\]{}:,;|+\-*/%<>^=¬∧∨↔∀∃≤≥≠→←⟨⟩·]|\S" +) + + +def normalize_lean_tokens(text: str) -> str: + """Canonical token-spacing normalizer so `(x: Nat)` and `(x : Nat)` compare equal.""" + toks: list[str] = [] + for tok in LEAN_TOKEN_RE.findall(text): + if tok == "->": + toks.append("→") + elif tok == "<-": + toks.append("←") + else: + toks.append(tok) + return " ".join(toks) + + +def extract_normalized_lean_defs(source: str) -> dict[str, str]: + """Extract `(short_def_name -> normalized_kw_sig_and_body)` from non-sorry Lean `def`/`abbrev` declarations.""" + cleaned = strip_lean_comments_and_strings(source) + defs: dict[str, str] = {} + for m in LEAN_DEF_OR_ABBREV_RE.finditer(cleaned): + rest = m.group("rest") + if re.search(r"\bsorry\b", rest): + continue + kw = m.group("kw") + short_name = m.group("name").split(".")[-1] + defs[short_name] = f"{kw} {normalize_lean_tokens(rest)}" + return defs + + +def verify_pinned_lean_defs(prompt_source: str, candidate_source: str) -> list[dict[str, Any]]: + """Reject candidate Lean modules that redefine a prompt-supplied target `def` or `abbrev` (`CertiFFI` hazard).""" + pinned = extract_normalized_lean_defs(prompt_source) + if not pinned: + return [] + cand_defs = extract_normalized_lean_defs(candidate_source) + findings: list[dict[str, Any]] = [] + for def_name, expected_norm in sorted(pinned.items()): + if def_name in cand_defs and cand_defs[def_name] != expected_norm: + findings.append({ + "line": 1, + "rule": "TARGET_DEF_BODY_MUTATED", + "message": f"candidate mutated pinned target definition '{def_name}' away from prompt specification", + }) + return findings + + +PROMPT_BACKTICK_THM_RE = re.compile( + r"`(?:theorem|lemma)\s+([A-Za-z_][A-Za-z0-9_.']*)\s*([^`]*:[^`]+)`" +) +PROMPT_BACKTICK_DEF_RE = re.compile( + r"`((?:def|abbrev)\s+[A-Za-z_][A-Za-z0-9_.']*[^`]+)`" +) +PROMPT_BACKTICK_TYPE_DECL_RE = re.compile( + r"`(?:structure|inductive)\s+([A-Za-z_][A-Za-z0-9_.']*)\b[^`]*`" +) +LEAN_NAMESPACE_RE = re.compile(r"(?m)^[ \t]*namespace[ \t]+([A-Za-z_][A-Za-z0-9_.']*)") +LEAN_DECL_NAME_RE_TEMPLATE = ( + rf"(?m)^[ \t]*{MODIFIERS_PAT}(?:def|abbrev|structure|inductive)[ \t]+" + r"(?:[A-Za-z_][A-Za-z0-9_.']*\.)?{short_name}\b" +) + +CHALLENGE_CMP_DRIVER_TEMPLATE = """import Lean + +open Lean Meta + +def runCheck (specMod candMod : Name) : CoreM Unit := do + let envSpec ← importModules #[{ module := specMod }] {} + let envCand ← importModules #[{ module := candMod }] {} + for (name, specInfo) in envSpec.constants.map₁.toList do + let some idx := envSpec.getModuleIdxFor? name | continue + let modName := envSpec.header.moduleNames[idx.toNat]! + unless modName == specMod do continue + if name.isInternal then continue + let strName := name.toString + if strName.endsWith ".rec" || strName.endsWith ".recOn" || strName.endsWith ".casesOn" || + strName.endsWith ".brecOn" || strName.endsWith ".below" || strName.endsWith ".ibelow" || + strName.endsWith ".noConfusion" || strName.endsWith ".noConfusionType" || + strName.endsWith ".ctorIdx" || strName.endsWith ".inj" || strName.endsWith ".injEq" || + strName.endsWith ".sizeOf_spec" || strName.endsWith ".elim" then + continue + let some candInfo := envCand.find? name + | IO.println s!"MCPROOF_CMP_ERR\\tMISSING_CHALLENGE_DECLARATION\\t{name}"; continue + match specInfo with + | .thmInfo st => + match candInfo with + | .thmInfo ct => + let (eqTy, _) ← (Meta.isDefEq st.type ct.type).run' {} + |>.toIO { fileName := "", fileMap := default } { env := envCand } + if !eqTy then + IO.println s!"MCPROOF_CMP_ERR\\tCHALLENGE_THEOREM_TYPE_MISMATCH\\t{name}" + | _ => + IO.println s!"MCPROOF_CMP_ERR\\tCHALLENGE_NOT_A_THEOREM\\t{name}" + | .defnInfo sd => + match candInfo with + | .defnInfo cd => + let (eqBoth, _) ← (do + let tOk ← Meta.isDefEq sd.type cd.type + if !tOk then return false + Meta.isDefEq sd.value cd.value + ).run' {} |>.toIO { fileName := "", fileMap := default } { env := envCand } + if !eqBoth then + IO.println s!"MCPROOF_CMP_ERR\\tCHALLENGE_DEF_MUTATED\\t{name}" + | _ => + IO.println s!"MCPROOF_CMP_ERR\\tCHALLENGE_NOT_A_DEF\\t{name}" + | .inductInfo si => + match candInfo with + | .inductInfo ci => + if si.ctors != ci.ctors || si.numParams != ci.numParams then + IO.println s!"MCPROOF_CMP_ERR\\tCHALLENGE_INDUCT_MUTATED\\t{name}" + | _ => + IO.println s!"MCPROOF_CMP_ERR\\tCHALLENGE_NOT_AN_INDUCT\\t{name}" + | _ => pure () + +#eval runCheck `McproofSpecMod `McproofCandMod +""" + + +def extract_prompt_lean_obligations(prompt_text: str, fenced_lean_blocks: Sequence[str] = ()) -> dict[str, Any]: + """Extract label-free theorem & definition obligations strictly from the prompt text (`NEON-C2`).""" + def_snippets: list[str] = list(fenced_lean_blocks) + required_defs: list[str] = [] + theorem_specs: list[tuple[str, str]] = [] + theorem_names: list[str] = [] + + for blk in fenced_lean_blocks: + cleaned_blk = strip_lean_comments_and_strings(blk) + for dm in LEAN_DEF_OR_ABBREV_RE.finditer(cleaned_blk): + dn = dm.group("name").split(".")[-1] + if dn not in required_defs: + required_defs.append(dn) + for tm in THEOREM_DECL_RE.finditer(cleaned_blk): + tn = tm.group("name").split(".")[-1] + hdr = tm.group("header").strip() + if tn not in theorem_names: + theorem_names.append(tn) + if ":" in hdr: + theorem_specs.append((tn, hdr)) + + for m_def in PROMPT_BACKTICK_DEF_RE.finditer(prompt_text): + snip = m_def.group(1).strip() + def_snippets.append(snip) + m_n = re.match(r"^(?:def|abbrev)\s+([A-Za-z_][A-Za-z0-9_.']*)\b", snip) + if m_n: + dn = m_n.group(1).split(".")[-1] + if dn not in required_defs: + required_defs.append(dn) + + for m_ty in PROMPT_BACKTICK_TYPE_DECL_RE.finditer(prompt_text): + tn_ty = m_ty.group(1).split(".")[-1] + if tn_ty not in required_defs: + required_defs.append(tn_ty) + + for m_thm in PROMPT_BACKTICK_THM_RE.finditer(prompt_text): + short_t = m_thm.group(1).split(".")[-1] + hdr = re.sub(r":=\s*(?:by\s+)?sorry\s*$", "", m_thm.group(2)).strip() + if short_t not in theorem_names: + theorem_names.append(short_t) + if ":" in hdr and not any(s[0] == short_t for s in theorem_specs): + theorem_specs.append((short_t, hdr)) + + for raw_t in re.findall(r"\b(?:theorem|lemma)\s+([A-Za-z_][A-Za-z0-9_.']*)\b", prompt_text): + st = raw_t.split(".")[-1] + if st not in theorem_names: + theorem_names.append(st) + + return { + "def_snippets": def_snippets, + "required_defs": required_defs, + "theorem_specs": theorem_specs, + "theorem_names": theorem_names, + } + + +def verify_lean4_prompt_obligations( + prompt_text: str, + candidate_source: str, + *, + fenced_prompt_lean: Sequence[str] = (), + lean_project_dir: Path | None = None, + run_kernel_type_check: bool = True, + timeout_sec: float = 60.0, +) -> list[dict[str, Any]]: + """Verify comment-stripped declarations AND kernel `Meta.isDefEq` theorem types against prompt text (`NEON-C2`).""" + obligations = extract_prompt_lean_obligations(prompt_text, fenced_prompt_lean) + cleaned_cand = strip_lean_comments_and_strings(candidate_source) + findings: list[dict[str, Any]] = [] + + declared_thms = {m.group("name").split(".")[-1] for m in THEOREM_DECL_RE.finditer(cleaned_cand)} + for req_thm in obligations["theorem_names"]: + if req_thm not in declared_thms: + findings.append({ + "line": 1, + "rule": "MISSING_PROMPT_THEOREM_NAME", + "message": f"Completion must declare and prove prompt theorem `{req_thm}` (not merely mention it in comments).", + }) + + for req_def in obligations["required_defs"]: + pat = LEAN_DECL_NAME_RE_TEMPLATE.format(short_name=re.escape(req_def)) + if not re.search(pat, cleaned_cand): + findings.append({ + "line": 1, + "rule": "MISSING_PROMPT_DEF_DECL", + "message": f"Completion omitted required prompt declaration `{req_def}`.", + }) + + for snip in obligations["def_snippets"]: + for df in verify_pinned_lean_defs(snip, candidate_source): + if not any(existing.get("message") == df.get("message") for existing in findings): + findings.append(df) + + if findings or not run_kernel_type_check or not obligations["theorem_specs"]: + return findings + + proj_dir = (lean_project_dir or (repo_root() / "lean")).resolve() + lean_bin = find_lean_tool("lean") + env = _lean_env() + lib_olean_dir = proj_dir / ".lake" / "build" / "lib" / "lean" + + cand_namespaces = LEAN_NAMESPACE_RE.findall(cleaned_cand) + open_lines = "\n".join(f"open {ns}" for ns in dict.fromkeys(cand_namespaces)) + ax_lines: list[str] = [] + pair_entries: list[str] = [] + for idx, (short_t, hdr) in enumerate(obligations["theorem_specs"]): + ax_name = f"_mcproof_pspec_{idx}" + ax_lines.append(f"axiom {ax_name} {hdr}") + qual_t = f"{cand_namespaces[0]}.{short_t}" if cand_namespaces else short_t + pair_entries.append(f"(`{ax_name}, `{qual_t}, `{short_t})") + + drv_src = ( + "import Lean\n" + "import McproofPromptCand\n" + "open Lean Meta\n" + f"{open_lines}\n" + + "\n".join(ax_lines) + + "\n\n#eval show CoreM Unit from do\n" + " let env ← getEnv\n" + f" let pairs : List (Name × Name × Name) := [{', '.join(pair_entries)}]\n" + " for (axN, fullN, shortN) in pairs do\n" + " let some specI := env.find? axN | IO.println s!\"MCPROOF_PSPEC_ERR\\tSPEC_ELAB_FAIL\\t{shortN}\"; continue\n" + " let candOpt := match env.find? fullN with\n" + " | some ci => some ci\n" + " | none => env.find? shortN\n" + " let some candI := candOpt | IO.println s!\"MCPROOF_PSPEC_ERR\\tMISSING_PROMPT_THEOREM_NAME\\t{shortN}\"; continue\n" + " match candI with\n" + " | .thmInfo ct =>\n" + " let (ok, _) ← (Meta.isDefEq specI.type ct.type).run' {}\n" + " |>.toIO { fileName := \"\", fileMap := default } { env := env }\n" + " if !ok then IO.println s!\"MCPROOF_PSPEC_ERR\\tPROMPT_THEOREM_TYPE_MISMATCH\\t{shortN}\"\n" + " | _ => IO.println s!\"MCPROOF_PSPEC_ERR\\tPROMPT_DECL_NOT_THEOREM\\t{shortN}\"\n" + ) + + with tempfile.TemporaryDirectory(prefix="mcproof_pspec_") as td: + tdp = Path(td).resolve() + cand_f = tdp / "McproofPromptCand.lean" + cand_o = tdp / "McproofPromptCand.olean" + drv_f = tdp / "McproofPromptSpecDriver.lean" + cand_f.write_text(candidate_source, encoding="utf-8") + drv_f.write_text(drv_src, encoding="utf-8") + + lean_path_parts = [str(tdp)] + if lib_olean_dir.is_dir(): + lean_path_parts.append(str(lib_olean_dir)) + sysroot_proc = subprocess.run([lean_bin, "--print-libdir"], capture_output=True, text=True, check=False) + if sysroot_proc.returncode == 0 and sysroot_proc.stdout.strip(): + lean_path_parts.append(sysroot_proc.stdout.strip()) + run_env = dict(env) + run_env["LEAN_PATH"] = os.pathsep.join(lean_path_parts) + + c_proc = subprocess.run( + [lean_bin, "-j", "1", "-D", "Elab.async=false", str(cand_f), "-o", str(cand_o)], + cwd=tdp, + env=run_env, + capture_output=True, + text=True, + timeout=timeout_sec, + check=False, + ) + if c_proc.returncode != 0: + return findings + + d_proc = subprocess.run( + [lean_bin, "-j", "1", "-D", "Elab.async=false", str(drv_f)], + cwd=tdp, + env=run_env, + capture_output=True, + text=True, + timeout=timeout_sec, + check=False, + ) + if d_proc.returncode != 0: + tail = " | ".join(ln.strip() for ln in ((d_proc.stdout or "") + "\n" + (d_proc.stderr or "")).splitlines()[-4:] if ln.strip()) + findings.append({ + "line": 1, + "rule": "PROMPT_THEOREM_TYPE_MISMATCH", + "message": f"Candidate theorem environment failed prompt statement check: {tail}", + }) + else: + for ln in (d_proc.stdout or "").splitlines(): + if ln.startswith("MCPROOF_PSPEC_ERR\t"): + parts = ln.split("\t") + rule_code = parts[1] if len(parts) > 1 else "PROMPT_THEOREM_TYPE_MISMATCH" + thm_id = parts[2] if len(parts) > 2 else "unknown" + findings.append({ + "line": 1, + "rule": rule_code, + "message": f"Candidate declaration `{thm_id}` does not match prompt theorem statement (`{rule_code}`).", + }) + return findings + + +def verify_lean4_against_challenge( + challenge_lean: str, + candidate_source: str, + *, + lean_project_dir: Path | None = None, + timeout_sec: float = 60.0, +) -> list[dict[str, Any]]: + """Verify via Lean `Meta.isDefEq` that `candidate_source` implements every def/inductive/theorem in `challenge_lean`.""" + proj_dir = (lean_project_dir or (repo_root() / "lean")).resolve() + lean_bin = find_lean_tool("lean") + env = _lean_env() + lib_olean_dir = proj_dir / ".lake" / "build" / "lib" / "lean" + findings: list[dict[str, Any]] = [] + + with tempfile.TemporaryDirectory(prefix="mcproof_chalcmp_") as td: + tdp = Path(td).resolve() + spec_f = tdp / "McproofSpecMod.lean" + spec_o = tdp / "McproofSpecMod.olean" + cand_f = tdp / "McproofCandMod.lean" + cand_o = tdp / "McproofCandMod.olean" + drv_f = tdp / "McproofChalCmpDriver.lean" + + spec_f.write_text(challenge_lean, encoding="utf-8") + cand_f.write_text(candidate_source, encoding="utf-8") + drv_f.write_text(CHALLENGE_CMP_DRIVER_TEMPLATE, encoding="utf-8") + + lean_path_parts = [str(tdp)] + if lib_olean_dir.is_dir(): + lean_path_parts.append(str(lib_olean_dir)) + sysroot_proc = subprocess.run([lean_bin, "--print-libdir"], capture_output=True, text=True, check=False) + if sysroot_proc.returncode == 0 and sysroot_proc.stdout.strip(): + lean_path_parts.append(sysroot_proc.stdout.strip()) + run_env = dict(env) + run_env["LEAN_PATH"] = os.pathsep.join(lean_path_parts) + + s_proc = subprocess.run( + [lean_bin, "-j", "1", "-D", "Elab.async=false", str(spec_f), "-o", str(spec_o)], + cwd=tdp, + env=run_env, + capture_output=True, + text=True, + timeout=timeout_sec, + check=False, + ) + c_proc = subprocess.run( + [lean_bin, "-j", "1", "-D", "Elab.async=false", str(cand_f), "-o", str(cand_o)], + cwd=tdp, + env=run_env, + capture_output=True, + text=True, + timeout=timeout_sec, + check=False, + ) + if s_proc.returncode != 0 or c_proc.returncode != 0: + findings.append({ + "line": 1, + "rule": "CHALLENGE_COMPARE_BUILD_FAILED", + "message": "Failed to compile challenge or candidate module for kernel statement comparison.", + }) + return findings + + cmp_proc = subprocess.run( + [lean_bin, "-j", "1", "-D", "Elab.async=false", str(drv_f)], + cwd=tdp, + env=run_env, + capture_output=True, + text=True, + timeout=timeout_sec, + check=False, + ) + if cmp_proc.returncode != 0: + tail = " | ".join(ln.strip() for ln in ((cmp_proc.stdout or "") + "\n" + (cmp_proc.stderr or "")).splitlines()[-4:] if ln.strip()) + findings.append({ + "line": 1, + "rule": "CHALLENGE_COMPARE_DRIVER_FAILED", + "message": f"Kernel challenge comparison driver failed: {tail}", + }) + return findings + + for ln in (cmp_proc.stdout or "").splitlines(): + if ln.startswith("MCPROOF_CMP_ERR\t"): + parts = ln.split("\t") + rule_code = parts[1] if len(parts) > 1 else "CHALLENGE_THEOREM_TYPE_MISMATCH" + decl_name = parts[2] if len(parts) > 2 else "unknown" + findings.append({ + "line": 1, + "rule": rule_code, + "message": f"Candidate failed challenge specification check on `{decl_name}` (`{rule_code}`).", + }) + return findings + + +def build_capped_lean_args( + lean_bin: str, + target_lean: Path, + *, + out_olean: Path | None = None, + max_memory_mib: int | None = 1400, +) -> list[str]: + """Construct single-threaded, synchronous, `--trust=0`, memory-capped Lean 4.35 invocation.""" + cmd = [lean_bin, "--trust=0", "-j", "1", "-D", "Elab.async=false"] + if max_memory_mib is not None and max_memory_mib > 0: + cmd.extend(["-M", str(int(max_memory_mib))]) + cmd.append(str(target_lean)) + if out_olean is not None: + cmd.extend(["-o", str(out_olean)]) + return cmd + + +def classify_host_resource_failure(returncode: int, combined_output: str) -> str | None: + """Return host-limit diagnostic reason when a Lean failure is OOM/signal/disk rather than mathematical.""" + low = combined_output.lower() + if returncode in (-9, 137): + return f"process killed by SIGKILL/OOM (rc={returncode})" + for marker in ( + "excessive memory consumption", + "memory allocation failed", + "out of memory", + "no space left on device", + "cannot allocate memory", + ): + if marker in low: + return f"host resource limit triggered ({marker})" + return None + + +def scrub_challenge_artifacts(project_dir: Path, challenge_module: str) -> list[str]: + """Unlink compiled `.olean`/`.ir` artifacts for `challenge_module` so `Spec` always compiles fresh.""" + if not challenge_module: + return [] + rel_parts = challenge_module.split(".") + removed: list[str] = [] + for base_sub in (Path(".lake/build/lib/lean"), Path(".lake/build/ir")): + stem_dir = project_dir / base_sub / Path(*rel_parts[:-1]) if len(rel_parts) > 1 else project_dir / base_sub + if not stem_dir.is_dir(): + continue + prefix = rel_parts[-1] + "." + for item in sorted(stem_dir.iterdir()): + if item.is_file() and item.name.startswith(prefix): + item.unlink() + removed.append(item.relative_to(project_dir).as_posix()) + return removed + + def lint_lean4_source( source: str, *, @@ -405,6 +992,7 @@ class LeanVerificationResult: findings: tuple[dict[str, Any], ...] caveats: tuple[str, ...] = () challenge_sorry_whitelisted_module: str | None = None + host_resource_exhausted: bool = False def to_dict(self) -> dict[str, Any]: return { @@ -418,6 +1006,7 @@ def to_dict(self) -> dict[str, Any]: "sandboxed": self.sandboxed, "caveats": list(self.caveats), "challenge_sorry_whitelisted_module": self.challenge_sorry_whitelisted_module, + "host_resource_exhausted": self.host_resource_exhausted, "findings": list(self.findings), } @@ -436,7 +1025,8 @@ def verify_lean4_source( require_theorem: bool = True, run_paranoid_leanchecker: bool = True, lean_project_dir: Path | None = None, - timeout_sec: int = 120, + timeout_sec: float = 120, + max_memory_mib: int | None = 1400, ) -> LeanVerificationResult: """Compile a Lean 4 source string, audit source escape hatches, verify kernel collectAxioms, and run leanchecker.""" proj_dir = (lean_project_dir or (repo_root() / "lean")).resolve() @@ -488,17 +1078,61 @@ def verify_lean4_source( run_env = dict(env) run_env["LEAN_PATH"] = os.pathsep.join(lean_path_parts) - comp = subprocess.run( - [lean_bin, str(cand_lean), "-o", str(cand_olean)], - cwd=tmp_dir, - env=run_env, - capture_output=True, - text=True, - timeout=timeout_sec, - check=False, + comp_cmd = build_capped_lean_args( + lean_bin, + cand_lean, + out_olean=cand_olean, + max_memory_mib=max_memory_mib, ) + try: + comp = subprocess.run( + comp_cmd, + cwd=tmp_dir, + env=run_env, + capture_output=True, + text=True, + timeout=timeout_sec, + check=False, + ) + except subprocess.TimeoutExpired as exc: + findings.append({ + "line": 1, + "rule": "HOST_RESOURCE_EXHAUSTED", + "message": f"Lean elaboration timed out after {timeout_sec}s ({exc})", + }) + return LeanVerificationResult( + passed=False, + toolchain=toolchain, + theorems_checked=(), + axioms_used=(), + nonstandard_axioms=(), + sorry_declarations=(), + paranoid_kernels_run=(), + sandboxed=False, + findings=tuple(findings), + host_resource_exhausted=True, + ) combined_out = (comp.stdout or "") + "\n" + (comp.stderr or "") if comp.returncode != 0: + host_reason = classify_host_resource_failure(comp.returncode, combined_out) + if host_reason is not None: + findings.append({ + "line": 1, + "rule": "HOST_RESOURCE_EXHAUSTED", + "message": host_reason, + }) + return LeanVerificationResult( + passed=False, + toolchain=toolchain, + theorems_checked=(), + axioms_used=(), + nonstandard_axioms=(), + sorry_declarations=(), + paranoid_kernels_run=(), + sandboxed=False, + findings=tuple(findings), + host_resource_exhausted=True, + ) tail = " | ".join(line.strip() for line in combined_out.strip().splitlines()[-6:] if line.strip()) findings.append({ "line": 1, @@ -726,6 +1360,9 @@ def verify_lean4_project_comparator( ) findings.extend(lint["findings"]) + # Scrub challenge `.olean`/`.ir` before `lake build` so `Spec` always compiles fresh from disk + scrub_challenge_artifacts(proj, challenge_mod) + # Run `lake build` build_res = subprocess.run( [lake_bin, "build"], diff --git a/src/mcproof/lineage.py b/src/mcproof/lineage.py index ce1d7f4..5fe72d8 100644 --- a/src/mcproof/lineage.py +++ b/src/mcproof/lineage.py @@ -46,6 +46,8 @@ "lean/lakefile.toml", "lean/comparator.json", "lean/Mcproof.lean", + "lean/Mcproof/TransitionBounds.lean", + "lean/Mcproof/AdapterBounds.lean", "lean/Mcproof/Spec.lean", "lean/Mcproof/Proofs.lean", ) diff --git a/src/mcproof/mutation.py b/src/mcproof/mutation.py index 02dfcda..b580f3f 100644 --- a/src/mcproof/mutation.py +++ b/src/mcproof/mutation.py @@ -2,6 +2,7 @@ from __future__ import annotations +import ast from dataclasses import dataclass import json import os @@ -17,6 +18,234 @@ KILL_FLOOR_NUM = 4 KILL_FLOOR_DEN = 5 # Exact 4/5 = 80% rational floor (checked via killed * 5 >= total * 4) +ALLOWLISTED_CHECKER_MODULES: frozenset[str] = frozenset( + { + "__future__", + "bisect", + "collections", + "copy", + "dataclasses", + "decimal", + "enum", + "fractions", + "functools", + "heapq", + "itertools", + "math", + "numbers", + "operator", + "re", + "string", + "typing", + } +) + +FORBIDDEN_CHECKER_BUILTINS: frozenset[str] = frozenset( + { + "__import__", + "breakpoint", + "compile", + "delattr", + "eval", + "exec", + "getattr", + "globals", + "input", + "locals", + "open", + "setattr", + "vars", + } +) + +FORBIDDEN_CHECKER_DUNDERS: frozenset[str] = frozenset( + { + "__bases__", + "__builtins__", + "__class__", + "__code__", + "__dict__", + "__globals__", + "__loader__", + "__mro__", + "__spec__", + "__subclasses__", + "ag_frame", + "cr_frame", + "f_back", + "f_builtins", + "f_code", + "f_globals", + "f_locals", + "gi_frame", + "tb_frame", + } +) + +FORBIDDEN_MODULE_HOP_ATTRS: frozenset[str] = frozenset( + { + "_builtins", + "_frozen_importlib", + "_imp", + "_io", + "_os", + "_posixsubprocess", + "_sys", + "builtins", + "codecs", + "importlib", + "io", + "modules", + "nt", + "open", + "os", + "pathlib", + "posix", + "shutil", + "socket", + "subprocess", + "sys", + } +) + + +def _is_negative_int_expr(node: ast.AST) -> bool: + if isinstance(node, ast.Constant) and type(node.value) is int and node.value < 0: + return True + if ( + isinstance(node, ast.UnaryOp) + and isinstance(node.op, ast.USub) + and isinstance(node.operand, ast.Constant) + and type(node.operand.value) is int + and node.operand.value > 0 + ): + return True + return False + + +def audit_exact_checker_ast( + source: str, + *, + strict_exact_math: bool = True, +) -> list[dict[str, Any]]: + """Fail-closed pure-math AST allowlist for candidate Python checkers (closes NEON-C1 & NEON-D1).""" + try: + tree = ast.parse(source) + except SyntaxError as exc: + return [ + { + "rule": "PYTHON_SYNTAX_ERROR", + "line": exc.lineno or 1, + "message": str(exc), + } + ] + + findings: list[dict[str, Any]] = [] + for node in ast.walk(tree): + line_no = getattr(node, "lineno", 1) or 1 + if isinstance(node, ast.Import): + for alias in node.names: + top = alias.name.split(".")[0] + if top not in ALLOWLISTED_CHECKER_MODULES: + findings.append( + { + "rule": "FORBIDDEN_CHECKER_IMPORT", + "line": line_no, + "message": f"disallowed module import {alias.name!r} in exact checker (pure-math allowlist only)", + } + ) + elif isinstance(node, ast.ImportFrom): + top = (node.module or "").split(".")[0] + if (node.level or 0) > 0 or top not in ALLOWLISTED_CHECKER_MODULES: + findings.append( + { + "rule": "FORBIDDEN_CHECKER_IMPORT", + "line": line_no, + "message": f"disallowed from-import {node.module or '.'!r} in exact checker (pure-math allowlist only)", + } + ) + else: + for alias in node.names: + imp_name = alias.name + if imp_name == "*" or imp_name.startswith("_") or imp_name in FORBIDDEN_MODULE_HOP_ATTRS: + findings.append( + { + "rule": "FORBIDDEN_CHECKER_IMPORT", + "line": line_no, + "message": f"disallowed symbol import {imp_name!r} from {node.module!r}", + } + ) + elif isinstance(node, ast.Attribute): + if node.attr in FORBIDDEN_CHECKER_DUNDERS or (node.attr.startswith("__") and node.attr.endswith("__")): + findings.append( + { + "rule": "FORBIDDEN_CHECKER_DUNDER", + "line": line_no, + "message": f"disallowed dunder attribute {node.attr!r} in exact checker", + } + ) + elif node.attr in FORBIDDEN_MODULE_HOP_ATTRS or node.attr in ( + "write_text", + "write_bytes", + "unlink", + "rmdir", + "mkdir", + "popen", + "system", + "urlopen", + ): + findings.append( + { + "rule": "FORBIDDEN_CHECKER_IO_ATTR", + "line": line_no, + "message": f"disallowed module-hop or host I/O attribute {node.attr!r} in exact checker", + } + ) + elif isinstance(node, ast.Name) and node.id in FORBIDDEN_CHECKER_BUILTINS: + findings.append( + { + "rule": "FORBIDDEN_CHECKER_BUILTIN", + "line": line_no, + "message": f"disallowed builtin reference {node.id!r} in exact checker", + } + ) + elif isinstance(node, ast.Call) and isinstance(node.func, ast.Name): + fn_id = node.func.id + if fn_id in FORBIDDEN_CHECKER_BUILTINS: + findings.append( + { + "rule": "FORBIDDEN_CHECKER_BUILTIN", + "line": line_no, + "message": f"disallowed builtin {fn_id!r} in exact checker", + } + ) + elif strict_exact_math and fn_id == "float": + findings.append( + { + "rule": "FLOAT_COERCION_CALL", + "line": line_no, + "message": "explicit float(...) coercion forbidden in exact int/Fraction checker", + } + ) + elif strict_exact_math and fn_id == "pow" and len(node.args) == 2 and _is_negative_int_expr(node.args[1]): + findings.append( + { + "rule": "NEGATIVE_POW_FLOAT_COERCION", + "line": line_no, + "message": "2-argument pow() with negative exponent produces float in exact checker", + } + ) + elif strict_exact_math and isinstance(node, ast.BinOp) and isinstance(node.op, ast.Pow): + if _is_negative_int_expr(node.right): + findings.append( + { + "rule": "NEGATIVE_POW_FLOAT_COERCION", + "line": line_no, + "message": "negative integer exponent `**` produces float in exact checker", + } + ) + return findings + def strict_python_env(extra: Mapping[str, str] | None = None) -> dict[str, str]: """Build deterministic single-threaded environment for python3 -O -B -W error execution.""" @@ -210,21 +439,48 @@ def _mutate_value(val: Any, kind: str) -> Any: _MUTATION_DRIVER_CODE = """from __future__ import annotations -import importlib.util -import json -import sys +import bisect, collections, copy, dataclasses, decimal, enum, fractions, functools +import heapq, importlib.util, io, itertools, json, math, numbers, operator, re, string, sys, typing from pathlib import Path +import builtins as _builtins + +_ALLOWLIST = frozenset({ + "__future__", "bisect", "collections", "copy", "dataclasses", "decimal", + "enum", "fractions", "functools", "heapq", "itertools", "math", "numbers", + "operator", "re", "string", "typing", +}) payload = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) mod_path = Path(payload["checker_path"]) fn_name = payload["entry_fn"] +result_path_str = str(payload["result_path"]) +_raw_open = io.open -spec = importlib.util.spec_from_file_location("candidate_checker_mod", mod_path) -if spec is None or spec.loader is None: - raise RuntimeError("cannot load candidate checker module") -mod = importlib.util.module_from_spec(spec) +mod_src = mod_path.read_text(encoding="utf-8") +code_obj = _builtins.compile(mod_src, str(mod_path), "exec") +mod = type(sys)("candidate_checker_mod") sys.modules["candidate_checker_mod"] = mod -spec.loader.exec_module(mod) + +_orig_import = _builtins.__import__ +def _guarded_import(name, globals=None, locals=None, fromlist=(), level=0): + if level and level > 0: + raise ImportError("relative imports are forbidden in candidate checker") + top = str(name).split(".")[0] + if top not in _ALLOWLIST: + raise ImportError(f"module {top!r} is not in pure-math checker allowlist") + return _orig_import(name, globals, locals, fromlist, level) + +_safe_b = dict(_builtins.__dict__) +_safe_b["__import__"] = _guarded_import +for _bad_b in ("open", "eval", "exec", "compile", "input", "breakpoint"): + _safe_b.pop(_bad_b, None) +mod.__dict__["__builtins__"] = _safe_b +_out_fh = _raw_open(result_path_str, "w", encoding="utf-8", newline="\\n") +def _audit_hook(ev, _args): + if ev in ("open", "os.system", "os.mkdir", "os.remove", "os.unlink", "os.rmdir", "os.rename") or ev.startswith(("subprocess.", "socket.")): + raise PermissionError(f"forbidden runtime event in exact checker: {ev}") +sys.addaudithook(_audit_hook) +_builtins.exec(code_obj, mod.__dict__) if not hasattr(mod, fn_name): raise AttributeError(f"candidate checker missing function {fn_name!r}") @@ -253,11 +509,10 @@ def _mutate_value(val: Any, kind: str) -> Any: except (ValueError, TypeError, KeyError, ArithmeticError, LookupError, RuntimeError) as exc: mutation_outcomes.append({"name": name, "killed": True, "returned": None, "error": f"{type(exc).__name__}: {exc}"}) -result_path = Path(payload["result_path"]) -result_path.write_text( - json.dumps({"valid_outcomes": valid_outcomes, "mutation_outcomes": mutation_outcomes}), - encoding="utf-8", -) +try: + _out_fh.write(json.dumps({"valid_outcomes": valid_outcomes, "mutation_outcomes": mutation_outcomes})) +finally: + _out_fh.close() """ @@ -306,6 +561,23 @@ def run_checker_mutation_suite( findings: list[dict[str, Any]] = [] floor_bps = (kill_floor_num * 10000) // kill_floor_den + ast_sec = audit_exact_checker_ast(checker_source, strict_exact_math=False) + if ast_sec: + findings.extend(ast_sec) + return MutationKillReport( + passed=False, + good_cases_total=len(valid_cases), + good_cases_passed=0, + mutations_total=0, + mutations_killed=0, + kill_fraction="0/0", + kill_rate_bps=0, + kill_floor_bps=floor_bps, + survived_mutations=(), + findings=tuple(findings), + python_sandboxed=False, + ) + if not valid_cases: findings.append({ "rule": "MISSING_VALID_CASES", diff --git a/src/mcproof/oracles.py b/src/mcproof/oracles.py index 8b98e6d..262d261 100644 --- a/src/mcproof/oracles.py +++ b/src/mcproof/oracles.py @@ -16,6 +16,7 @@ from .lean_harness import verify_lean4_project_comparator, verify_lean4_source from .lineage import canonical_json_bytes, verify_lineage_manifest from .mutation import ( + audit_exact_checker_ast, run_candidate_python_confined, run_checker_mutation_suite, run_f0_gate_and_corruptions, @@ -168,10 +169,11 @@ def run_lean4_oracle( timeout_sec=timeout_sec, ) + status_str = "PROVEN" if res.passed else ("INCOMPLETE_HOST_LIMIT" if res.host_resource_exhausted else "REJECTED") return OracleReceipt.create( oracle="lean4", passed=res.passed, - epistemic_status="PROVEN" if res.passed else "REJECTED", + epistemic_status=status_str, checks=res.to_dict(), findings=res.findings, ) @@ -294,6 +296,7 @@ def run_checker_oracle( planted_mutations: Sequence[Mapping[str, Any]] | None = None, strict_exact_math: bool = True, force_cargo_run: bool = False, + allow_static_lint_only: bool = False, ) -> OracleReceipt: """Audit a Python exact-math checker with soko checker_auditor, mde guard --code-audit, python3 -O -B -W error, and >= 80% mutation kill floor.""" checker_mod = _load_dep_module("soko", "soko_mcp/checker_auditor.py", "soko_checker_auditor") @@ -337,14 +340,48 @@ def run_checker_oracle( "message": str(f["detail"]), }) - # Explicitly scan for `float(` calls in exact-math checkers (in addition to float literals & `/`) + # Fail-closed pure-math AST import/builtin/float/negative-pow allowlist (closes NEON-C1 & NEON-D1) + ast_allowlist_findings = audit_exact_checker_ast(checker_source, strict_exact_math=strict_exact_math) + seen_rules_lines = {(f.get("rule"), f.get("line", 0)) for f in findings} + for af in ast_allowlist_findings: + key = (af.get("rule"), af.get("line", 0)) + if key not in seen_rules_lines: + seen_rules_lines.add(key) + findings.append(af) + if strict_exact_math and re.search(r"\bfloat\s*\(", checker_source): - findings.append({ - "rule": "FLOAT_COERCION_CALL", - "line": 1, - "message": "explicit float(...) coercion forbidden in exact int/Fraction checker", - }) + if not any(f.get("rule") == "FLOAT_COERCION_CALL" for f in findings): + findings.append({ + "rule": "FLOAT_COERCION_CALL", + "line": 1, + "message": "explicit float(...) coercion forbidden in exact int/Fraction checker", + }) + + _SMOKE_GUARD_SCRIPT = ( + "import bisect, collections, copy, dataclasses, decimal, enum, fractions, functools, " + "heapq, itertools, math, numbers, operator, pathlib, re, string, sys, typing, builtins as _b; " + "_AL = frozenset({'__future__','bisect','collections','copy','dataclasses','decimal','enum'," + "'fractions','functools','heapq','itertools','math','numbers','operator','re','string','typing'}); " + "_co = _b.compile(pathlib.Path(sys.argv[1]).read_text(encoding='utf-8'), sys.argv[1], 'exec'); " + "_m = type(sys)('cand_smoke_mod'); sys.modules['cand_smoke_mod'] = _m; " + "_oi = _b.__import__; _sb = dict(_b.__dict__); " + "_sb['__import__'] = lambda n, g=None, l=None, f=(), lv=0: (_ for _ in ()).throw(ImportError(n)) " + "if (lv and lv > 0) or str(n).split('.')[0] not in _AL else _oi(n, g, l, f, lv); " + "[_sb.pop(k, None) for k in ('open','eval','exec','compile','input','breakpoint')]; " + "_m.__dict__['__builtins__'] = _sb; " + "sys.addaudithook(lambda ev, _a: (_ for _ in ()).throw(PermissionError(ev)) " + "if ev in ('open','os.system','os.mkdir','os.remove','os.unlink','os.rmdir','os.rename') " + "or ev.startswith(('subprocess.','socket.')) else None); " + "_b.exec(_co, _m.__dict__)" + ) + + non_exec_blocking_rules = { + f["rule"] + for f in findings + if f["rule"] not in ("VACUOUS_CONSTANT_TRUE_RETURN", "bare_assert", "IDENTICAL_ALPHA_SKELETON") + } + strict_import_ok = False with tempfile.TemporaryDirectory(prefix="mcproof_chk_") as tmp_str: tmp_py = Path(tmp_str) / "checker.py" tmp_py.write_text(checker_source, encoding="utf-8") @@ -355,24 +392,29 @@ def run_checker_oracle( "line": f.get("line", 0), "message": f["message"], }) - - # Verify module imports cleanly under confined `python3 -I -O -B -W error` - smoke_proc, _, _ = run_candidate_python_confined( - ["-c", "import runpy, sys; runpy.run_path(sys.argv[1])", str(tmp_py)], - writable_dir=Path(tmp_str), - timeout_sec=30, - sandbox_mode="auto", - ) - strict_import_ok = smoke_proc is not None and smoke_proc.returncode == 0 - if not strict_import_ok: - err_msg = "" if smoke_proc is None else (smoke_proc.stderr or smoke_proc.stdout).strip()[:240] - findings.append({ - "rule": "STRICT_PYTHON_IMPORT_ERROR", - "message": f"failed under `python3 -I -O -B -W error`: {err_msg}", - }) + if f["rule"] != "bare_assert": + non_exec_blocking_rules.add(f["rule"]) + + # Never execute untrusted candidate Python when security/float/syntax static audit failed, + # or when static-lint-only mode already rejected the candidate (NEON-C1). + allow_exec = (len(non_exec_blocking_rules) == 0) and (bool(valid_cases) or len(findings) == 0) + if allow_exec: + smoke_proc, _, _ = run_candidate_python_confined( + ["-c", _SMOKE_GUARD_SCRIPT, str(tmp_py)], + writable_dir=Path(tmp_str), + timeout_sec=30, + sandbox_mode="auto", + ) + strict_import_ok = smoke_proc is not None and smoke_proc.returncode == 0 + if not strict_import_ok: + err_msg = "" if smoke_proc is None else (smoke_proc.stderr or smoke_proc.stdout).strip()[:240] + findings.append({ + "rule": "STRICT_PYTHON_IMPORT_ERROR", + "message": f"failed under `python3 -I -O -B -W error`: {err_msg}", + }) mut_dict: dict[str, Any] | None = None - if valid_cases: + if valid_cases and len(non_exec_blocking_rules) == 0: mut_rep = run_checker_mutation_suite( checker_source, entry_fn=entry_fn, @@ -382,6 +424,10 @@ def run_checker_oracle( mut_dict = mut_rep.to_dict() mut_passed = mut_rep.passed findings.extend(mut_rep.findings) + elif valid_cases: + mut_passed = False + elif allow_static_lint_only: + mut_passed = True else: findings.append({ "rule": "MISSING_VALID_CASES", @@ -396,10 +442,11 @@ def run_checker_oracle( and mut_passed and len(findings) == 0 ) + ok_tier = "LINT_PASS" if (not valid_cases and allow_static_lint_only) else "VERIFIED" return OracleReceipt.create( oracle="exact_checker", passed=passed, - epistemic_status="VERIFIED" if passed else "REJECTED", + epistemic_status=ok_tier if passed else "REJECTED", checks={ "soko_checker_audit": dual_rep, "mde_code_audit_passed": mde_rep.passed, diff --git a/src/mcproof/repair_loop.py b/src/mcproof/repair_loop.py new file mode 100644 index 0000000..d2c52e9 --- /dev/null +++ b/src/mcproof/repair_loop.py @@ -0,0 +1,1396 @@ +"""Targeted Plane 1 section retrieval and label-free multi-turn Tier 2 repair loop (`mcproof#5`). + +Implements three architectural guarantees: +1. Targeted Plane 1 section retrieval (`<= 1,500` tokens matched to Family A-D, prompt keywords, + and prior-turn failure rules, with automatic chunk rotation when stagnation is detected). +2. Strict separation between the **label-free inference-time oracle** (`default_inference_oracle`, + which sees ONLY `family` and `prompt` and drives Turn `1..k` diagnostics and early stopping with + zero access to `expected_tier`, `valid_cases`, `planted_mutations`, or ground-truth harness files) + and the **post-hoc benchmark grader** (`eval_grader_fn`, which scores deployed outputs at budgets + `k = 1..max_turns` without ever feeding ground-truth labels or verdicts back to the model). +3. Completion/AST hash (`compute_candidate_state_hash`) and `(rule, line)` failure-signature + (`compute_failure_signature`) stagnation detection across turns `1..k`, switching Plane 1 + reference chunks and structural framing whenever `temperature = 0.0` repeats a visited state. +""" + +from __future__ import annotations + +import ast +import base64 +from dataclasses import dataclass +from decimal import Decimal +from fractions import Fraction +import hashlib +import json +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import time +from typing import Any, Callable, Mapping, Sequence +import urllib.request + +from .lean_harness import ( + THEOREM_DECL_RE, + strip_lean_comments_and_strings, + verify_lean4_against_challenge, + verify_lean4_prompt_obligations, + verify_pinned_lean_defs, +) +from .oracles import ( + FENCED_BLOCK_RE, + OracleReceipt, + SHA40_RE, + extract_fenced_blocks, + grade_family_completion, + run_checker_oracle, + run_claim_hygiene_oracle, + run_lean4_oracle, + run_prose_and_mixed_zone_oracle, + run_verdicts_oracle, + sanitize_verdict_id, +) +from .paths import dep_repo, repo_root + +MAX_RETRIEVAL_TOKENS_HARD_CAP = 1500 +QWEN35_STOP_TOKENS: tuple[str, ...] = ("<|im_end|>", "<|endoftext|>") +WORD_TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*|[0-9]+|[^\sA-Za-z0-9_]") +HEX_LITERAL_RE = re.compile(r"\b[0-9a-fA-F]{16,64}\b") +QUOTED_PAYLOAD_RE = re.compile(r"('[^'\n]{12,}'|\"[^\"\n]{12,}\")") +DICT_LIST_LITERAL_RE = re.compile(r"(\{[^{}\n]{8,}\}|\[[^\[\]\n]{12,}\])") +EXPECTED_COMPARISON_RE = re.compile(r"(?i)\b(!=\s*expected\b|expected[ _-]*tier\s*[:=]?\s*['\"]?[A-Z_]+|expected[ _-]*verdict\s*[:=]?\s*['\"]?[A-Z_]+)") +LEAN_THM_NAME_RE = re.compile(r"\b(?:theorem|lemma)\s+([A-Za-z_][A-Za-z0-9_.']*)\b") +PY_DEF_NAME_RE = re.compile(r"\bdef\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(") + + +@dataclass(frozen=True) +class Plane1Chunk: + """Single retrievable subsection of `spec/plane1_formal_spec.md`.""" + + chunk_id: str + section_num: int + heading: str + body: str + est_tokens: int + keywords: frozenset[str] + + +def estimate_tokens_conservative(text: str) -> int: + """Conservative upper-bound token count safe for Qwen3.5 and cl100k_base without network calls.""" + if not text: + return 0 + utf8_len = len(text.encode("utf-8")) + lex_pieces = len(WORD_TOKEN_RE.findall(text)) + by_bytes = (utf8_len + 2) // 3 + est = max(lex_pieces, by_bytes) + try: + import tiktoken # type: ignore[import-not-found] + + enc = tiktoken.get_encoding("cl100k_base") + est = max(est, int(len(enc.encode(text)) * 1.08) + 4) + except ImportError: + est = est + 4 + return est + + +def parse_plane1_chunks(spec_path: Path | None = None) -> list[Plane1Chunk]: + """Split `spec/plane1_formal_spec.md` along `##` / `###` / `####` headings into retrievable chunks.""" + target = spec_path or (repo_root() / "spec" / "plane1_formal_spec.md") + raw_lines = target.read_text(encoding="utf-8").splitlines() + + chunks: list[Plane1Chunk] = [] + cur_sec_num = 0 + cur_h2 = "" + cur_heading = "Overview" + cur_buf: list[str] = [] + + def _flush() -> None: + text = "\n".join(cur_buf).strip() + if not text: + return + full_text = f"{cur_heading}\n{text}" if not text.startswith("#") else text + words = {w.lower() for w in re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}", full_text)} + cid = f"sec{cur_sec_num}_{len(chunks):02d}" + chunks.append( + Plane1Chunk( + chunk_id=cid, + section_num=cur_sec_num, + heading=f"{cur_h2} / {cur_heading}".strip(" /"), + body=full_text, + est_tokens=estimate_tokens_conservative(full_text), + keywords=frozenset(words), + ) + ) + + for line in raw_lines: + m2 = re.match(r"^##\s+(\d+)\.\s+(.*)$", line) + if m2: + _flush() + cur_buf = [line] + cur_sec_num = int(m2.group(1)) + cur_h2 = f"{cur_sec_num}. {m2.group(2).strip()}" + cur_heading = cur_h2 + continue + if line.startswith("### ") or line.startswith("#### "): + _flush() + cur_buf = [line] + cur_heading = line.lstrip("#").strip() + continue + cur_buf.append(line) + _flush() + return chunks + + +FAMILY_PRIMARY_SECTIONS: dict[str, tuple[int, ...]] = { + "A": (9, 4, 12, 13), + "B": (5, 6, 11, 12), + "C": (8, 7, 3), + "D": (10, 7, 9), +} + +RULE_TO_SECTION_BOOST: dict[str, tuple[int, ...]] = { + "FORBIDDEN_SORRY": (9,), + "FORBIDDEN_ADMIT": (9,), + "FORBIDDEN_AXIOM": (9,), + "FORBIDDEN_NATIVE_DECIDE": (9,), + "LEAN_COMPILE": (9, 12, 13), + "WEAK_STATEMENT": (4,), + "VACUOUS": (4,), + "MISSING_THEOREM": (9, 4), + "FLOAT": (6,), + "BARE_ASSERT": (5, 6), + "SKELETON": (5,), + "NON_VACUOUS": (5, 11), + "MISSING_SECTION": (7, 10), + "CLAIM_TIER": (8, 7), + "INFLATED": (8,), + "PROMPT_EVIDENCE_CONTRADICTION": (8, 7), + "VERDICT": (8,), + "F0_": (8, 3), + "MISSING_DATA_EIDO_SKIP": (10,), + "PROSE": (10, 7), +} + + +def retrieve_plane1_excerpt( + *, + family: str, + prompt_text: str, + prior_findings: Sequence[Mapping[str, Any]] = (), + max_tokens: int = 1250, + spec_path: Path | None = None, + exclude_chunk_ids: Sequence[str] = (), + prefer_worked_examples: bool = False, +) -> dict[str, Any]: + """Retrieve a focused `<= 1,500`-token Plane 1 excerpt tailored to `family`, prompt, and errors.""" + budget = min(max(250, int(max_tokens)), MAX_RETRIEVAL_TOKENS_HARD_CAP) + fam = family.strip().upper() + if fam not in FAMILY_PRIMARY_SECTIONS: + raise ValueError(f"unsupported family {family!r} (expected A, B, C, or D)") + + excluded = set(exclude_chunk_ids) + chunks = parse_plane1_chunks(spec_path) + prompt_words = {w.lower() for w in re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}", prompt_text)} + + boosted_sections: dict[int, int] = {} + for rank_idx, sec in enumerate(FAMILY_PRIMARY_SECTIONS[fam]): + boosted_sections[sec] = max(boosted_sections.get(sec, 0), 40 - rank_idx * 8) + + if prefer_worked_examples: + boosted_sections[12] = boosted_sections.get(12, 0) + 32 + boosted_sections[13] = boosted_sections.get(13, 0) + 32 + + error_words: set[str] = set() + for f in prior_findings: + rule_up = str(f.get("rule", "")).upper() + error_words.update(w.lower() for w in re.findall(r"[A-Za-z]{3,}", rule_up)) + for pat, secs in RULE_TO_SECTION_BOOST.items(): + if pat in rule_up: + for s_num in secs: + boosted_sections[s_num] = boosted_sections.get(s_num, 0) + 35 + + scored: list[tuple[int, int, Plane1Chunk]] = [] + for idx, ch in enumerate(chunks): + if ch.est_tokens > budget: + continue + sec_score = boosted_sections.get(ch.section_num, 0) + kw_overlap = len(prompt_words & ch.keywords) * 4 + len(error_words & ch.keywords) * 6 + if prefer_worked_examples: + style_bonus = 18 if any(w in ch.heading for w in ("Reference Proof", "Pattern", "Worked", "Algorithm")) else 0 + repeat_penalty = -30 if ch.chunk_id in excluded else 0 + else: + style_bonus = 12 if any(w in ch.heading for w in ("Checklist", "Template", "Forbidden", "Rules", "Playbook", "Matrix")) else 0 + repeat_penalty = -10 if ch.chunk_id in excluded else 0 + total_score = sec_score + kw_overlap + style_bonus + repeat_penalty + if total_score > 0: + scored.append((total_score, -idx, ch)) + + scored.sort(reverse=True) + candidate_pool = ( + [entry for entry in scored if entry[2].chunk_id not in excluded] + if excluded + else list(scored) + ) + if not candidate_pool: + candidate_pool = list(scored) + + header = f"### Plane 1 Reference Rules (Family {fam})\n" + used_tokens = estimate_tokens_conservative(header) + 12 + selected: list[Plane1Chunk] = [] + seen_ids: set[str] = set() + + for _score, _neg_idx, ch in candidate_pool: + if ch.chunk_id in seen_ids: + continue + sep_cost = 4 + if used_tokens + ch.est_tokens + sep_cost > budget: + continue + selected.append(ch) + seen_ids.add(ch.chunk_id) + used_tokens += ch.est_tokens + sep_cost + + selected.sort(key=lambda c: c.chunk_id) + excerpt_body = header + "\n\n".join(c.body for c in selected) + final_est = estimate_tokens_conservative(excerpt_body) + + while final_est > MAX_RETRIEVAL_TOKENS_HARD_CAP and selected: + selected.pop() + excerpt_body = header + "\n\n".join(c.body for c in selected) + final_est = estimate_tokens_conservative(excerpt_body) + + return { + "family": fam, + "excerpt": excerpt_body, + "text": excerpt_body, + "context": excerpt_body, + "estimated_tokens": final_est, + "max_tokens_cap": MAX_RETRIEVAL_TOKENS_HARD_CAP, + "chunk_ids": [c.chunk_id for c in selected], + "headings": [c.heading for c in selected], + "rotated_for_stagnation": prefer_worked_examples, + } + + +def retrieve_plane1_sections( + family: str, + prompt: str = "", + findings: Sequence[Mapping[str, Any]] = (), + spec_path: Path | None = None, + max_tokens: int = 1250, + exclude_chunk_ids: Sequence[str] = (), + prefer_worked_examples: bool = False, +) -> dict[str, Any]: + """Bridge-compatible Plane 1 retriever (`<= 1,500` tokens) returning `text` and metadata.""" + return retrieve_plane1_excerpt( + family=family, + prompt_text=prompt, + prior_findings=findings, + max_tokens=max_tokens, + spec_path=spec_path, + exclude_chunk_ids=exclude_chunk_ids, + prefer_worked_examples=prefer_worked_examples, + ) + + +# Stagnation State Deduplication (AST / Normalized Hash + Failure Signature) -- + +def _strip_lean_comments(src: str) -> str: + return " ".join(strip_lean_comments_and_strings(src).split()) + + +def compute_candidate_state_hash(family: str, completion_text: str) -> str: + """Compute a formatting-invariant AST or token-normalized SHA-256 for stagnation detection.""" + fam = family.strip().upper() + if fam == "B": + py_blocks = extract_fenced_blocks(completion_text, ("python", "py")) + py_src = py_blocks[-1] if py_blocks else completion_text + try: + tree = ast.parse(py_src) + norm = ast.dump(tree, include_attributes=False) + return hashlib.sha256(norm.encode("utf-8")).hexdigest() + except SyntaxError: + return hashlib.sha256(" ".join(py_src.split()).encode("utf-8")).hexdigest() + + if fam in ("A", "D"): + lean_blocks = extract_fenced_blocks(completion_text, ("lean", "lean4")) + if lean_blocks: + lean_norm = "\n".join(_strip_lean_comments(b) for b in lean_blocks) + prose_norm = " ".join(completion_text.split()) if fam == "D" else "" + return hashlib.sha256((lean_norm + "\n" + prose_norm).encode("utf-8")).hexdigest() + + return hashlib.sha256(" ".join(completion_text.split()).encode("utf-8")).hexdigest() + + +def compute_failure_signature(findings: Sequence[Mapping[str, Any]]) -> str: + """Canonical SHA-256 over sorted `(rule, line)` pairs to detect deterministic error plateaus.""" + if not findings: + return "PASS" + pairs = sorted( + { + (str(f.get("rule", "FINDING")).upper(), int(f["line"]) if ("line" in f and f["line"] is not None) else -1) + for f in findings + } + ) + raw = json.dumps(pairs, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(raw).hexdigest()[:24] + + +# Zero-Leakage Diagnostic Sanitizer & Label-Free Inference Oracle ------------- + +def sanitize_oracle_diagnostics( + family: str, + findings: Sequence[Mapping[str, Any]], + *, + max_items: int = 8, +) -> list[dict[str, str]]: + """Extract minimal structured Tier 2 diagnostics while scrubbing fixtures and expected labels.""" + fam = family.strip().upper() + cleaned: list[dict[str, str]] = [] + + for raw in list(findings)[:max_items]: + rule = str(raw.get("rule", "ORACLE_FINDING")).strip() or "ORACLE_FINDING" + if rule.upper() in ("EXPECTED_TIER_MISMATCH", "VERDICT_MISMATCH", "F0_TIER_BUCKET_MISMATCH"): + # Defense in depth: never surface ground-truth expected label comparisons even if passed in + continue + msg = str(raw.get("message", "")).strip() + line_no = raw.get("line") + + # Strip any accidental `!= expected '...'` substring and prompt-injection fences/mentions + msg = EXPECTED_COMPARISON_RE.sub("", msg) + msg = msg.replace("```", "'''") + msg = re.sub(r"@([A-Za-z0-9_-]+)", r"at_\1", msg) + msg = msg.replace("", msg) + msg = QUOTED_PAYLOAD_RE.sub("", msg) + msg = HEX_LITERAL_RE.sub("", msg) + if "valid_case" in rule.lower() or "valid case" in msg.lower(): + m_cnt = re.search(r"(\d+\s*/\s*\d+)", msg) + ratio_note = f" ({m_cnt.group(1)})" if m_cnt else "" + msg = f"Checker rejected valid specification test case{ratio_note}; check exact integer/Fraction invariants." + elif "mutation" in rule.lower() or "planted" in msg.lower() or "kill" in msg.lower(): + m_cnt = re.search(r"(\d+\s*/\s*\d+)", msg) + ratio_note = f" ({m_cnt.group(1)})" if m_cnt else "" + msg = f"Checker failed >=80% planted-bug mutation kill floor{ratio_note}; verify all certificate conditions independently." + else: + msg = re.sub(r"(/[A-Za-z0-9._-]+){2,}/([A-Za-z0-9._-]+\.(?:lean|py|md|json))", r"\2", msg) + + msg_one_line = " ".join(msg.split())[:260] + entry: dict[str, str] = {"rule": rule, "diagnostic": msg_one_line} + if line_no is not None: + entry["line"] = str(line_no) + cleaned.append(entry) + + return cleaned + + +def format_diagnostics( + findings: Sequence[Mapping[str, Any]], + *, + family: str = "", +) -> str: + """Format sanitized, zero-fixture-leak Tier 2 diagnostics into compact multi-turn repair text.""" + inferred_fam = family.strip().upper() + if not inferred_fam: + joined_rules = " ".join(str(f.get("rule", "")) for f in findings).upper() + if any(k in joined_rules for k in ("MUTATION", "VALID_CASE", "FLOAT", "SKELETON", "CHECKER", "ENTRY_FN")): + inferred_fam = "B" + elif "EIDO_SKIP" in joined_rules or "EMBEDDED_LEAN" in joined_rules: + inferred_fam = "D" + elif any(k in joined_rules for k in ("CLAIM", "VERDICT", "F0_", "BAR_", "PROMPT_EVIDENCE")): + inferred_fam = "C" + else: + inferred_fam = "A" + sanitized = sanitize_oracle_diagnostics(inferred_fam, findings) + if not sanitized: + return "Tier 2 deterministic oracle rejected the candidate with unspecified error." + lines = [f"Tier 2 deterministic oracle diagnostics (Family {inferred_fam}, {len(sanitized)} item(s)):"] + for idx, d in enumerate(sanitized, start=1): + loc = f" (line {d['line']})" if "line" in d else "" + lines.append(f"{idx}. [{d['rule']}]{loc}: {d['diagnostic']}") + return "\n".join(lines) + + +def _check_prompt_shas_preserved(prompt_text: str, completion_text: str) -> list[dict[str, Any]]: + """Verify any 40-char commit SHA stated in the prompt is not truncated or dropped when SHAs are required.""" + prompt_shas = sorted({m.lower() for m in SHA40_RE.findall(prompt_text)}) + if not prompt_shas: + return [] + comp_shas = {m.lower() for m in SHA40_RE.findall(completion_text)} + missing = [s for s in prompt_shas if s not in comp_shas] + if missing: + return [ + { + "rule": "MISSING_PROMPT_COMMIT_SHA40", + "message": "Completion omitted or truncated the 40-hex source commit SHA given in the prompt.", + } + ] + return [] + + +def _audit_family_c_inference_only(prompt_text: str, completion_text: str) -> dict[str, Any]: + """100% label-free Family C inference check using ONLY prompt text and public schema validators.""" + lean_blocks = extract_fenced_blocks(completion_text, ("lean", "lean4")) + has_valid_lean = False + findings: list[dict[str, Any]] = [] + if lean_blocks: + l_rec = run_lean4_oracle("\n\n".join(lean_blocks), strict_init_only=True) + has_valid_lean = l_rec.passed + if not l_rec.passed: + findings.extend(l_rec.findings) + + # Label-free claim hygiene (`expected_tier=None`) + hyg = run_claim_hygiene_oracle( + completion_text, + expected_tier=None, + has_kernel_proof=has_valid_lean, + ) + findings.extend(hyg.findings) + declared_tier = str(hyg.checks.get("declared_tier") or "").upper() + + # Check prompt-visible epistemic contradictions without reading `expected_tier` + p_low = prompt_text.lower() + prompt_signals_unproved = any( + sig in p_low + for sig in ( + "uses `sorry`", + "contains `sorry`", + "with `sorry`", + "admit", + "native_decide", + "no formal proof", + "without a proof", + "heuristic", + "conjecture", + "sampled", + "monte carlo", + ) + ) + if prompt_signals_unproved and declared_tier == "PROVEN" and not has_valid_lean: + findings.append( + { + "rule": "PROMPT_EVIDENCE_CONTRADICTION", + "message": "Prompt describes unproved, heuristic, or sorry-containing evidence; cannot claim PROVEN.", + } + ) + if ('"stale": true' in p_low or "status = \"in-progress\"" in p_low or "no specification approval" in p_low) and re.search( + r"(?im)^status\s*:\s*verified\b", completion_text + ): + findings.append( + { + "rule": "PROMPT_STATUS_INFLATION", + "message": "Prompt indicates in-progress or stale status without approval; cannot declare 'Status: verified'.", + } + ) + + # Validate any fenced JSON block against public verdict / F0 schemas (zero expected-value lookup) + json_blocks = extract_fenced_blocks(completion_text, ("json",)) + if ("```json" in prompt_text or "verdict" in p_low or "chart_bucket" in p_low) and not json_blocks: + if "json" in p_low: + findings.append( + { + "rule": "MISSING_REQUIRED_JSON_BLOCK", + "message": "Prompt requests a structured ```json block.", + } + ) + for jb in json_blocks: + try: + parsed = json.loads(jb) + except ValueError as exc: + findings.append({"rule": "INVALID_JSON_BLOCK", "message": str(exc)}) + continue + if not isinstance(parsed, dict): + continue + if "verdict" in parsed and "source_commit" in parsed: + raw_id = parsed.get("id", "verdict_01") + vid = sanitize_verdict_id(raw_id) + if vid is None: + findings.append({"rule": "INVALID_VERDICT_ID", "message": f"unsafe verdict id: {raw_id!r}"}) + else: + with tempfile.TemporaryDirectory(prefix="mcproof_inf_v_") as td: + vf = Path(td).resolve() / f"{vid}.json" + vf.write_text(jb, encoding="utf-8", newline="\n") + v_rep = run_verdicts_oracle(vf) + if not v_rep.passed: + findings.extend(v_rep.findings) + findings.extend(_check_prompt_shas_preserved(prompt_text, jb)) + elif "record" in parsed or "chart_bucket" in parsed: + rec_d = parsed.get("record") + bucket = parsed.get("chart_bucket") + if not isinstance(rec_d, dict) or bucket not in ("claimed", "verified"): + findings.append( + { + "rule": "F0_SCHEMA_ERROR", + "message": "F0 JSON wrapper requires 'record' object and 'chart_bucket' in {'claimed', 'verified'}.", + } + ) + else: + if bucket == "claimed" and rec_d.get("verify") is not None: + findings.append( + { + "rule": "F0_CLAIMED_HAS_VERIFY_BLOCK", + "message": "chart_bucket='claimed' records must not carry an unverified 'verify' block.", + } + ) + if bucket == "verified" and rec_d.get("stale") is True: + findings.append( + { + "rule": "F0_STALE_IN_VERIFIED_BUCKET", + "message": "stale=True records belong in quarantine, never chart_bucket='verified'.", + } + ) + kn = rec_d.get("kappa_num") + kd = rec_d.get("kappa_den") + kdec = rec_d.get("kappa_decimal") + if isinstance(kn, int) and isinstance(kd, int) and kd > 0 and isinstance(kdec, str): + exact_rat = Decimal(kn) / Decimal(kd) + try: + given_dec = Decimal(kdec) + if abs(exact_rat - given_dec) > Decimal("1e-14"): + findings.append( + { + "rule": "F0_KAPPA_DECIMAL_MISMATCH", + "message": "kappa_decimal does not match exact kappa_num / kappa_den to 15 significant digits.", + } + ) + except ArithmeticError: + findings.append({"rule": "F0_INVALID_KAPPA_DECIMAL", "message": "unparseable kappa_decimal"}) + + passed = len(findings) == 0 + return { + "passed": passed, + "epistemic_status": declared_tier if (passed and declared_tier) else ("VERIFIED" if passed else "REJECTED"), + "findings": findings, + } + + +def default_inference_oracle( + item_public_view: Mapping[str, Any], + completion_text: str, +) -> dict[str, Any]: + """100% label-free inference-time Tier 2 oracle (`mcproof#5`). + + Reads ONLY `item_public_view['family']` and `item_public_view['prompt']`. + Never accesses `expected_tier`, `valid_cases`, `planted_mutations`, `harness_path`, + `target_path`, `expected_verdict`, or `expected_f0_tier`. + """ + fam = str(item_public_view["family"]).strip().upper() + prompt_text = str(item_public_view.get("prompt", "")) + + if fam == "A": + lean_blocks = extract_fenced_blocks(completion_text, ("lean", "lean4")) + lean_src = "\n\n".join(lean_blocks) if lean_blocks else completion_text + rec = run_lean4_oracle( + lean_src, + strict_init_only=True, + check_vacuous=True, + require_theorem=True, + ) + findings = [dict(f) for f in rec.findings] + prompt_lean_blocks = extract_fenced_blocks(prompt_text, ("lean", "lean4")) + if prompt_lean_blocks: + findings.extend(verify_pinned_lean_defs("\n\n".join(prompt_lean_blocks), lean_src)) + findings.extend( + verify_lean4_prompt_obligations( + prompt_text, + lean_src, + fenced_prompt_lean=prompt_lean_blocks, + run_kernel_type_check=rec.passed, + ) + ) + passed = rec.passed and len(findings) == 0 + return { + "passed": passed, + "epistemic_status": "PROVEN" if passed else "REJECTED", + "findings": findings, + } + + if fam == "B": + py_blocks = extract_fenced_blocks(completion_text, ("python", "py")) + py_src = py_blocks[-1] if py_blocks else completion_text + prompt_py_blocks = extract_fenced_blocks(prompt_text, ("python", "py")) + gen_spec_src = prompt_py_blocks[0] if prompt_py_blocks else None + + prose_outside_fences = FENCED_BLOCK_RE.sub("", prompt_text) + verify_cands = re.findall(r"\b(verify[A-Za-z0-9_]*)\s*\(", prose_outside_fences) + check_cands = re.findall(r"\b(check[A-Za-z0-9_]*)\s*\(", prose_outside_fences) + expected_fn = verify_cands[0] if verify_cands else (check_cands[0] if check_cands else "verify") + + rec_b = run_checker_oracle( + py_src, + generator_or_spec_source=gen_spec_src, + entry_fn=expected_fn, + valid_cases=(), + planted_mutations=None, + strict_exact_math=True, + allow_static_lint_only=True, + ) + findings_b = [dict(f) for f in rec_b.findings] + if not re.search(rf"\bdef\s+{re.escape(expected_fn)}\s*\(", py_src): + findings_b.append( + { + "rule": "MISSING_ENTRY_FUNCTION", + "message": f"Python checker must define entry function `def {expected_fn}(...):`.", + } + ) + passed_b = rec_b.passed and len(findings_b) == 0 + return { + "passed": passed_b, + "epistemic_status": "LINT_PASS" if passed_b else "REJECTED", + "findings": findings_b, + } + + if fam == "C": + return _audit_family_c_inference_only(prompt_text, completion_text) + + if fam == "D": + lean_blocks = extract_fenced_blocks(completion_text, ("lean", "lean4")) + if not lean_blocks: + return { + "passed": False, + "epistemic_status": "REJECTED", + "findings": [{"rule": "MISSING_EMBEDDED_LEAN_BLOCK", "message": "Family D response must include a fenced ```lean block."}], + } + mixed = run_prose_and_mixed_zone_oracle( + completion_text, + require_formal_sections=True, + require_data_eido_skip=True, + verify_embedded_lean=True, + verify_verdict_shas=True, + ) + findings_d = [dict(f) for f in mixed.findings] + findings_d.extend(_check_prompt_shas_preserved(prompt_text, completion_text)) + passed_d = mixed.passed and len(findings_d) == 0 + return { + "passed": passed_d, + "epistemic_status": mixed.epistemic_status if passed_d else "REJECTED", + "findings": findings_d, + } + + raise ValueError(f"unsupported prompt family {fam!r} (expected A, B, C, or D)") + + +def build_repair_turn_prompt( + *, + family: str, + original_prompt: str, + plane1_excerpt: str, + turn_index: int, + prior_completion: str | None = None, + sanitized_diagnostics: Sequence[Mapping[str, str]] = (), + stagnation_detected: bool = False, + refuted_routes: Sequence[Mapping[str, Any]] = (), +) -> str: + """Construct the bounded user prompt for turn `turn_index` (`1`-indexed).""" + fam = family.strip().upper() + format_directive = { + "A": "Return a complete stock-`Init` Lean 4.35.0 solution inside a ```lean fenced block with zero `sorry`, `admit`, `axiom`, or `native_decide`.", + "B": "Return a standalone Python 3 exact-math checker inside a ```python fenced block using `int`/`fractions.Fraction` only, zero `float`, zero `/` true-division, and zero bare `assert`.", + "C": "Return the audited formal response with explicit honest tier (`PROVEN`, `VERIFIED`, `KNOWN`, `CONJECTURE`, or `REFUSED`) and non-empty `## What is not proved` and `## What you must trust` sections.", + "D": "Return the mixed-register Markdown document keeping `
...
` intact, a valid ```lean proof block, plain-language prose (no em dashes, no outcome forecasts, paragraphs <=420 chars), and `## What is not proved` + `## What you must trust`.", + }[fam] + + if turn_index <= 1 or not sanitized_diagnostics: + return ( + f"{plane1_excerpt}\n\n" + f"### Task\n{original_prompt.strip()}\n\n" + f"### Output Contract\n{format_directive}" + ) + + diag_lines = "\n".join( + f"- [{d['rule']}" + (f":L{d['line']}" if "line" in d else "") + f"] {d['diagnostic']}" + for d in sanitized_diagnostics + ) + + if stagnation_detected: + # State-deduplication escape: omit verbatim broken draft so deterministic decoding does not copy it back! + reset_strategy = { + "A": "Do NOT repeat the previous proof script. State explicit helper lemmas over `Nat`/`Int` using structural `induction` + `omega` + `Nat.mul_add`/`Int.add_assoc` instead of unavailable Mathlib tactics.", + "B": "Do NOT repeat the previous checker skeleton. Construct a clean, independent certificate verifier checking types (`isinstance(v, int) and not isinstance(v, bool)`), domain bounds, and exact `Fraction` equations from scratch.", + "C": "Do NOT repeat the previous draft. Re-derive the claim tier directly from the evidence in the prompt, verify every 40-hex SHA and JSON field verbatim, and write concrete `## What is not proved` and `## What you must trust` bullets.", + "D": "Do NOT repeat the previous draft. Emit short plain paragraphs (<=350 chars, zero em dashes or semicolons), `
` around formal metadata, a self-contained stock-`Init` ```lean proof, and both boundary sections.", + }[fam] + obstruction_note = "" + if refuted_routes: + ob_lines = "\n".join( + f"- Turn {r['turn']} (`state={r['state_hash']}`, primary rule `{r['primary_rule']}`)" + for r in list(refuted_routes)[-4:] + ) + obstruction_note = f"### Refuted Routes (Do Not Re-Explore)\n{ob_lines}\n\n" + return ( + f"{plane1_excerpt}\n\n" + f"### Task\n{original_prompt.strip()}\n\n" + f"### Stagnation Escape Alert (Turn {turn_index})\n" + f"Your previous attempt reproduced an already-rejected AST or identical error signature:\n{diag_lines}\n\n" + f"{obstruction_note}" + f"### Fresh Structural Synthesis Plan\n{reset_strategy} {format_directive}" + ) + + prev_trimmed = (prior_completion or "").strip() + if len(prev_trimmed) > 2200: + prev_trimmed = prev_trimmed[:2200] + "\n..." + + return ( + f"{plane1_excerpt}\n\n" + f"### Task\n{original_prompt.strip()}\n\n" + f"### Previous Attempt (Turn {turn_index - 1})\n{prev_trimmed}\n\n" + f"### Deterministic Tier 2 Oracle Diagnostics\n{diag_lines}\n\n" + f"### Repair Instructions\nFix every Tier 2 diagnostic above and emit the complete corrected response. {format_directive}" + ) + + +def format_qwen35_chat_prompt(user_prompt: str, system_prompt: str | None = None) -> str: + """Render deterministic Qwen3.5 chat template (`enable_thinking = False`).""" + parts: list[str] = [] + if system_prompt: + parts.append(f"<|im_start|>system\n{system_prompt}<|im_end|>\n") + parts.append(f"<|im_start|>user\n{user_prompt}<|im_end|>\n") + parts.append("<|im_start|>assistant\n\n\n\n\n") + return "".join(parts) + + +def make_llama_server_generator( + server_url: str, + *, + lora: Sequence[Mapping[str, Any]] = (), + max_tokens: int = 768, + seed: int = 42, + slot_id: int = 0, + timeout_s: float = 300.0, +) -> Callable[[str, int], dict[str, Any]]: + """Create a deterministic `temperature = 0.0, top_p = 1.0` generator backed by `llama-server`.""" + base = server_url.rstrip("/") + + def _post_json(url: str, payload: Mapping[str, Any]) -> dict[str, Any]: + req = urllib.request.Request( + url, + data=json.dumps(dict(payload)).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=timeout_s) as resp: + raw = resp.read().decode("utf-8") + return json.loads(raw) if raw.strip() else {} + + def _generate(user_prompt: str, _turn_idx: int) -> dict[str, Any]: + slot_erase_err: str | None = None + try: + _post_json(f"{base}/slots/{slot_id}?action=erase", {}) + except OSError as exc: + slot_erase_err = str(exc) + rendered = format_qwen35_chat_prompt(user_prompt) + req_body: dict[str, Any] = { + "prompt": rendered, + "n_predict": max_tokens, + "temperature": 0.0, + "top_p": 1.0, + "seed": seed, + "cache_prompt": False, + "stop": list(QWEN35_STOP_TOKENS), + "lora": [dict(x) for x in lora], + "id_slot": slot_id, + } + t0 = time.perf_counter() + resp = _post_json(f"{base}/completion", req_body) + elapsed_ms = round((time.perf_counter() - t0) * 1000.0, 2) + content = str(resp.get("content", "")) + prompt_toks = int(resp.get("tokens_evaluated") or estimate_tokens_conservative(rendered)) + comp_toks = int(resp.get("tokens_predicted") or estimate_tokens_conservative(content)) + out: dict[str, Any] = { + "content": content, + "prompt_tokens": prompt_toks, + "completion_tokens": comp_toks, + "elapsed_ms": elapsed_ms, + } + if slot_erase_err is not None: + out["slot_erase_warning"] = slot_erase_err + return out + + return _generate + + +# Post-Hoc Benchmark Grader (`eval_grader_fn`, NEVER fed back to model) ------- + +def _extract_first_json_block(text: str) -> str | None: + blocks = extract_fenced_blocks(text, ("json",)) + return blocks[0].strip() if blocks else None + + +def _run_public_family_c_harness(harness: Mapping[str, Any], completion_text: str) -> list[dict[str, Any]]: + """Post-hoc Family C ground-truth scorer matching `datasets/scripts/grade_item.py`.""" + otype = harness.get("oracle_type") + meta = harness.get("spec_meta") or {} + findings: list[dict[str, Any]] = [] + + if otype == "bar_check": + bar_tpl = dep_repo("mission") / "bar" / "template" + bar_chk = dep_repo("mission") / "bar" / "check.py" + pkg = str(meta["pkg"]) + thms = list(meta["theorem_names"]) + readme_body = re.sub(r"(?im)^Claim-Tier:[^\n]*\n+", "", completion_text, count=1) + says_verified = bool( + re.search(r"(?im)^status\s*:\s*verified\b", readme_body) + and not re.search(r"(?im)^status\s*:.*not verified", readme_body) + ) + bar_status = "verified" if says_verified else str(meta["bar_status"]) + with tempfile.TemporaryDirectory(prefix="mcproof_rep_bar_") as td: + root = Path(td) + shutil.copytree(bar_tpl, root, dirs_exist_ok=True) + (root / "Clamp.lean").unlink(missing_ok=True) + shutil.rmtree(root / "Clamp", ignore_errors=True) + (root / f"{pkg}.lean").write_text( + f"namespace {pkg}\ndef op (x : Nat) : Nat := x\nend {pkg}\n", + encoding="utf-8", + ) + (root / pkg).mkdir(exist_ok=True) + spec_lines = [f"import {pkg}", f"namespace {pkg}.Spec"] + proof_lines = [f"import {pkg}", f"namespace {pkg}.Spec"] + for tfull in thms: + tshort = str(tfull).split(".")[-1] + spec_lines.append(f"theorem {tshort} (x : Nat) : {pkg}.op x = x := by sorry") + proof_lines.append(f"theorem {tshort} (x : Nat) : {pkg}.op x = x := rfl") + spec_lines.append(f"end {pkg}.Spec\n") + proof_lines.append(f"end {pkg}.Spec\n") + (root / pkg / "Spec.lean").write_text("\n".join(spec_lines), encoding="utf-8") + (root / pkg / "Proofs.lean").write_text("\n".join(proof_lines), encoding="utf-8") + (root / "Main.lean").write_text( + f'import {pkg}\ndef main : IO Unit := IO.println s!"{{{pkg}.op 1}}"\n', + encoding="utf-8", + ) + (root / "lakefile.toml").write_text( + f'name = "{pkg.lower()}"\nversion = "0.1.0"\ndefaultTargets = ["{pkg}"]\n\n' + f'[[lean_lib]]\nname = "{pkg}"\n\n[[lean_lib]]\nname = "{pkg}.Spec"\n\n' + f'[[lean_lib]]\nname = "{pkg}.Proofs"\n\n[[lean_exe]]\nname = "{pkg.lower()}"\nroot = "Main"\n', + encoding="utf-8", + ) + (root / "comparator.json").write_text( + json.dumps( + { + "challenge_module": f"{pkg}.Spec", + "solution_module": f"{pkg}.Proofs", + "theorem_names": thms, + "permitted_axioms": ["propext", "Quot.sound", "Classical.choice"], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + (root / "bar.toml").write_text( + f'[project]\nname = "{pkg.lower()}"\nstatus = "{bar_status}"\npage = "README.md"\n' + f'spec_authors = ["example-author"]\nentry_points = ["Main.lean"]\n\n' + f'[spec]\nfiles = ["SPEC.md", "{pkg}/Spec.lean"]\ncode = ["{pkg}.op"]\n\n' + f'[tests]\nresults = "tests/TEST-RESULTS.md"\ncommand = "{meta["test_cmd"]}"\nfiles = ["tests/run_tests.py"]\n', + encoding="utf-8", + ) + (root / "README.md").write_text(readme_body, encoding="utf-8") + out_json = root / "bar_out.json" + proc = subprocess.run( + [sys.executable, "-O", "-B", "-W", "error", str(bar_chk), str(root), "--no-lean", "--offline", "--json", str(out_json)], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + findings.append({"rule": "BAR_CHECK_FAILED", "message": "bar check failed"}) + elif out_json.is_file(): + rep = json.loads(out_json.read_text(encoding="utf-8")) + by_item = {r["item"]: r["status"] for r in rep.get("items", [])} + for req_i in (2, 4, 5, 6): + if by_item.get(req_i) != "PASS": + findings.append({"rule": "BAR_CHECK_ITEM_FAIL", "message": f"bar item {req_i} did not pass"}) + return findings + + if otype == "verdict_audit": + jb = _extract_first_json_block(completion_text) + if jb is None: + return [{"rule": "MISSING_VERDICT_JSON", "message": "missing ```json verdict block"}] + try: + obj = json.loads(jb) + except ValueError as exc: + return [{"rule": "INVALID_VERDICT_JSON", "message": str(exc)}] + if obj.get("verdict") != meta.get("expected_verdict"): + findings.append({"rule": "VERDICT_MISMATCH", "message": "verdict value mismatch"}) + if obj.get("source_commit") != meta.get("source_commit"): + findings.append({"rule": "SOURCE_COMMIT_MISMATCH", "message": "source_commit does not match pinned 40-char SHA"}) + if obj.get("axioms") != meta.get("expected_axioms"): + findings.append({"rule": "AXIOMS_LIST_MISMATCH", "message": "axioms list mismatch"}) + return findings + + if otype == "f0_claim_gate": + jb = _extract_first_json_block(completion_text) + if jb is None: + return [{"rule": "MISSING_F0_JSON", "message": "missing ```json F0 claim wrapper"}] + try: + payload = json.loads(jb) + except ValueError as exc: + return [{"rule": "INVALID_F0_JSON", "message": str(exc)}] + rec = payload.get("record") + bucket = payload.get("chart_bucket") + if not isinstance(rec, dict) or bucket not in ("claimed", "verified"): + return [{"rule": "INVALID_F0_WRAPPER", "message": "invalid F0 wrapper"}] + f0_src = dep_repo("f0") + with tempfile.TemporaryDirectory(prefix="mcproof_rep_f0_") as td: + rdir = Path(td) + (rdir / "data").mkdir() + (rdir / "docs").mkdir() + shutil.copytree(f0_src / "scripts", rdir / "scripts") + shutil.copyfile(f0_src / "feed.schema.json", rdir / "feed.schema.json") + shutil.copyfile(f0_src / "feed.schema.json", rdir / "docs" / "feed.schema.json") + art_bytes = str(meta["cert_artifact_bytes_utf8"]).encode("utf-8") + art_sha = str(meta["cert_artifact_sha256"]) + (rdir / "data" / "certificate.artifact.json").write_bytes(art_bytes) + mirror_obj = { + "kappa": "90446448750933/125000000000000000", + "decimal": "0.000723571590007464", + "certificate": "certificates/selected-result.json", + "certificate_sha256": art_sha, + } + mirror_bytes = (json.dumps(mirror_obj, indent=2, sort_keys=True) + "\n").encode("utf-8") + (rdir / "data" / "selected-result.json").write_bytes(mirror_bytes) + blob_sha1 = hashlib.sha1(b"blob %d\x00" % len(mirror_bytes) + mirror_bytes).hexdigest() + raw_cap = { + "path": "certificates/selected-result.json", + "sha": blob_sha1, + "size": len(mirror_bytes), + "content": base64.b64encode(mirror_bytes).decode("ascii"), + } + (rdir / "data" / "selected-result.raw.json").write_text(json.dumps(raw_cap, indent=2) + "\n", encoding="utf-8") + (rdir / "data" / "records.json").write_text(json.dumps({"records": [rec]}, indent=2, sort_keys=True) + "\n", encoding="utf-8") + gen_at = "2026-11-19T00:00:00Z" + main_sha = hashlib.sha256(b"upstream-main-anchor").hexdigest()[:40] + feed_obj = { + "feed_version": 1, + "generated_at": gen_at, + "upstream_main_sha": main_sha, + "selected_result": { + "kappa": mirror_obj["kappa"], + "decimal": mirror_obj["decimal"], + "certificate_sha256": art_sha, + "selected_result_sha256": hashlib.sha256(mirror_bytes).hexdigest(), + "source_blob_sha": blob_sha1, + "source_path": "certificates/selected-result.json", + }, + "records": [rec] if not rec.get("stale") else [], + "quarantined": [rec] if rec.get("stale") else [], + } + c_claimed = { + "pr": rec.get("pr"), + "head_sha": rec.get("head_sha"), + "kappa_decimal": rec.get("kappa_decimal"), + "draft": rec.get("draft", False), + "pr_updated_at": rec.get("pr_updated_at"), + "pr_url": (rec.get("source_refs") or {}).get("pr_url"), + } + c_ver = { + "pr": rec.get("pr"), + "head_sha": rec.get("head_sha"), + "kappa_decimal": rec.get("kappa_decimal"), + "tier": rec.get("tier"), + "pr_updated_at": rec.get("pr_updated_at"), + "pr_url": (rec.get("source_refs") or {}).get("pr_url"), + } + chart_obj = { + "generated_at": gen_at, + "claimed": [c_claimed] if bucket == "claimed" else [], + "verified": [c_ver] if bucket == "verified" else [], + } + (rdir / "docs" / "feed.json").write_text(json.dumps(feed_obj, indent=2, sort_keys=True) + "\n", encoding="utf-8") + (rdir / "docs" / "chart-data.json").write_text(json.dumps(chart_obj, indent=2, sort_keys=True) + "\n", encoding="utf-8") + proc = subprocess.run( + [sys.executable, "-O", "-B", "-W", "error", str(rdir / "scripts" / "gate.py"), "--root", str(rdir)], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + findings.append({"rule": "F0_GATE_FAILED", "message": "f0 gate check failed"}) + if rec.get("tier") != meta.get("expected_f0_tier") or bucket != meta.get("expected_chart_bucket"): + findings.append({"rule": "F0_TIER_BUCKET_MISMATCH", "message": "f0 tier or bucket mismatch"}) + return findings + + return findings + + +def default_dataset_item_grader( + item: Mapping[str, Any], + completion_text: str, + *, + datasets_root: Path | None = None, +) -> dict[str, Any]: + """Post-hoc ground-truth Tier 2 benchmark scorer (`eval_grader_fn`).""" + ds_root = datasets_root or (repo_root() / "datasets") + fam = str(item["family"]).strip().upper() + exp_tier = item.get("expected_tier") + harness_rel = item.get("harness_path") + harness: dict[str, Any] = {} + if harness_rel and (ds_root / str(harness_rel)).is_file(): + harness = json.loads((ds_root / str(harness_rel)).read_text(encoding="utf-8")) + + gen_src = harness.get("generator_source") if fam == "B" else None + valid_cases = list(harness.get("valid_cases", [])) if fam == "B" else [] + planted = list(harness.get("planted_mutations")) if (fam == "B" and "planted_mutations" in harness) else None + + rec: OracleReceipt = grade_family_completion( + fam, + completion_text, + expected_tier=str(exp_tier) if exp_tier else None, + generator_source=gen_src, + valid_cases=valid_cases, + planted_mutations=planted, + ) + passed = rec.passed + findings = [dict(f) for f in rec.findings] + + if fam == "A": + lean_blocks = extract_fenced_blocks(completion_text, ("lean", "lean4")) + lean_src = "\n\n".join(lean_blocks) if lean_blocks else completion_text + cleaned_lean = strip_lean_comments_and_strings(lean_src) + declared_thms = {m.group("name").split(".")[-1] for m in THEOREM_DECL_RE.finditer(cleaned_lean)} + if harness.get("theorem_name"): + thm_name = str(harness["theorem_name"]) + short_name = thm_name.split(".")[-1] + if short_name not in declared_thms: + passed = False + findings.append( + { + "rule": "MISSING_REQUIRED_THEOREM_NAME", + "message": f"Completion must declare and prove theorem `{thm_name}` without renaming it.", + } + ) + if passed and harness.get("challenge_lean"): + chal_findings = verify_lean4_against_challenge(str(harness["challenge_lean"]), lean_src) + if chal_findings: + passed = False + findings.extend(chal_findings) + elif item.get("prompt"): + p_findings = verify_lean4_prompt_obligations( + str(item["prompt"]), + lean_src, + run_kernel_type_check=passed, + ) + if p_findings: + passed = False + findings.extend(p_findings) + + if fam == "C" and harness: + c_findings = _run_public_family_c_harness(harness, completion_text) + if c_findings: + passed = False + findings.extend(c_findings) + + return { + "passed": passed, + "epistemic_status": rec.epistemic_status if passed else "REJECTED", + "findings": findings, + } + + +def run_repair_item( + item: Mapping[str, Any], + generator_fn: Callable[[str, int], dict[str, Any] | str], + *, + max_turns: int = 4, + retrieval_max_tokens: int = 1250, + inference_oracle_fn: Callable[[Mapping[str, Any], str], Mapping[str, Any]] | None = None, + eval_grader_fn: Callable[[Mapping[str, Any], str], Mapping[str, Any]] | None = None, + grader_fn: Callable[[Mapping[str, Any], str], Mapping[str, Any]] | None = None, + datasets_root: Path | None = None, + spec_path: Path | None = None, + include_plane1_retrieval: bool = True, +) -> dict[str, Any]: + """Execute bounded `k`-turn label-free inference repair + post-hoc benchmark scoring on one item. + + - `inference_oracle_fn` (default: `default_inference_oracle`) sees ONLY `{'id', 'family', 'prompt'}` + and drives Turn `1..k` diagnostics and early stopping (`inference_certified_at_turn`). + - `eval_grader_fn` (or `grader_fn` alias; default: `default_dataset_item_grader`) scores each + deployed candidate post-hoc for `pass_at_1..k` without ever leaking ground-truth labels or + private fixture verdicts back into the repair prompt or stopping rule. + """ + if max_turns < 1: + raise ValueError("max_turns must be >= 1") + + item_id = str(item["id"]) + fam = str(item["family"]).strip().upper() + prompt_text = str(item["prompt"]) + public_view: dict[str, Any] = {"id": item_id, "family": fam, "prompt": prompt_text} + + active_inf_oracle = inference_oracle_fn or default_inference_oracle + active_eval_grader = eval_grader_fn or grader_fn or ( + lambda it, comp: default_dataset_item_grader(it, comp, datasets_root=datasets_root) + ) + + prior_inf_findings: list[dict[str, Any]] = [] + prior_sanitized: list[dict[str, str]] = [] + prior_completion: str | None = None + seen_state_hashes: set[str] = set() + seen_fail_sigs: set[str] = set() + seen_chunk_ids: list[str] = [] + refuted_routes: list[dict[str, Any]] = [] + stagnation_next = False + stagnation_events = 0 + + turn_records: list[dict[str, Any]] = [] + eval_pass_by_turn: dict[int, bool] = {} + total_prompt_tokens = 0 + total_completion_tokens = 0 + total_retrieved_tokens = 0 + total_elapsed_ms = 0.0 + inference_certified_at_turn: int | None = None + deployed_eval_passed_when_certified: bool | None = None + + for turn_idx in range(1, max_turns + 1): + if include_plane1_retrieval: + ret = retrieve_plane1_excerpt( + family=fam, + prompt_text=prompt_text, + prior_findings=prior_inf_findings, + max_tokens=retrieval_max_tokens, + spec_path=spec_path, + exclude_chunk_ids=seen_chunk_ids if stagnation_next else (), + prefer_worked_examples=stagnation_next, + ) + excerpt = str(ret["excerpt"]) + ret_toks = int(ret["estimated_tokens"]) + chunk_ids = list(ret["chunk_ids"]) + for cid in chunk_ids: + if cid not in seen_chunk_ids: + seen_chunk_ids.append(cid) + else: + excerpt = "" + ret_toks = 0 + chunk_ids = [] + + user_prompt = build_repair_turn_prompt( + family=fam, + original_prompt=prompt_text, + plane1_excerpt=excerpt, + turn_index=turn_idx, + prior_completion=prior_completion, + sanitized_diagnostics=prior_sanitized, + stagnation_detected=stagnation_next, + refuted_routes=refuted_routes, + ) + + gen_raw = generator_fn(user_prompt, turn_idx) + if isinstance(gen_raw, str): + comp_text = gen_raw + p_toks = estimate_tokens_conservative(user_prompt) + c_toks = estimate_tokens_conservative(comp_text) + gen_ms = 0.0 + else: + comp_text = str(gen_raw.get("content", "")) + p_toks = int(gen_raw.get("prompt_tokens") or estimate_tokens_conservative(user_prompt)) + c_toks = int(gen_raw.get("completion_tokens") or estimate_tokens_conservative(comp_text)) + gen_ms = float(gen_raw.get("elapsed_ms") or 0.0) + + # 1. Label-free inference-time oracle (drives diagnostics & stopping) + t_inf = time.perf_counter() + inf_res = active_inf_oracle(public_view, comp_text) + inf_ms = round((time.perf_counter() - t_inf) * 1000.0, 2) + inf_passed = bool(inf_res.get("passed", False)) + inf_findings = [dict(f) for f in inf_res.get("findings", ())] + sanitized = sanitize_oracle_diagnostics(fam, inf_findings) + + # 2. Post-hoc ground-truth evaluation (NEVER fed back into diagnostics or stopping!) + t_eval = time.perf_counter() + eval_res = active_eval_grader(item, comp_text) + eval_ms = round((time.perf_counter() - t_eval) * 1000.0, 2) + eval_passed = bool(eval_res.get("passed", False)) + eval_pass_by_turn[turn_idx] = eval_passed + + # 3. Candidate AST / state deduplication & failure-signature plateau check + state_hash = compute_candidate_state_hash(fam, comp_text) + fail_sig = compute_failure_signature(inf_findings) + repeated_state = state_hash in seen_state_hashes + repeated_fail_sig = (not inf_passed) and (fail_sig in seen_fail_sigs) + stagnation_hit = (not inf_passed) and (repeated_state or repeated_fail_sig) + if stagnation_hit: + stagnation_events += 1 + seen_state_hashes.add(state_hash) + if not inf_passed: + seen_fail_sigs.add(fail_sig) + + total_prompt_tokens += p_toks + total_completion_tokens += c_toks + total_retrieved_tokens += ret_toks + total_elapsed_ms += gen_ms + inf_ms + + turn_records.append( + { + "turn": turn_idx, + "inference_passed": inf_passed, + "eval_passed": eval_passed, + "passed": eval_passed, + "stagnation_prompt_mode": stagnation_next, + "stagnation_detected_after_turn": stagnation_hit, + "candidate_state_hash": state_hash[:24], + "failure_signature": fail_sig, + "retrieved_plane1_tokens": ret_toks, + "retrieved_chunk_ids": chunk_ids, + "prompt_tokens": p_toks, + "completion_tokens": c_toks, + "generation_ms": round(gen_ms, 2), + "inference_oracle_ms": inf_ms, + "eval_grader_ms": eval_ms, + "response_sha256": hashlib.sha256(comp_text.encode("utf-8")).hexdigest(), + "diagnostic_rules": [d["rule"] for d in sanitized], + "diagnostic_count": len(inf_findings), + } + ) + + # Stopping rule depends ONLY on label-free `inf_passed`! + if inf_passed: + inference_certified_at_turn = turn_idx + deployed_eval_passed_when_certified = eval_passed + break + + primary_rule = sanitized[0]["rule"] if sanitized else "ORACLE_REJECTED" + refuted_routes.append( + { + "turn": turn_idx, + "state_hash": state_hash[:12], + "primary_rule": primary_rule, + } + ) + prior_completion = comp_text + prior_inf_findings = inf_findings + prior_sanitized = sanitized + stagnation_next = stagnation_hit + + # Deployed output at budget k (1..max_turns): + # - If inference oracle certified at turn t <= k, the deployed output at budget k is turn t's output! + # - Otherwise the loop ran through turn k, so the deployed output at budget k is turn k's output. + pass_by_k: dict[str, bool] = {} + first_eval_pass_turn: int | None = None + for k in range(1, max_turns + 1): + if inference_certified_at_turn is not None and inference_certified_at_turn <= k: + deployed_ok = bool(deployed_eval_passed_when_certified) + else: + deployed_ok = bool(eval_pass_by_turn.get(min(k, len(turn_records)), False)) + pass_by_k[f"pass_at_{k}"] = deployed_ok + if deployed_ok and first_eval_pass_turn is None: + first_eval_pass_turn = k + + final_deployed_passed = pass_by_k[f"pass_at_{max_turns}"] + return { + "id": item_id, + "family": fam, + "max_turns": max_turns, + "label_free_inference_stopping": True, + "passed": final_deployed_passed, + "inference_certified": inference_certified_at_turn is not None, + "inference_certified_at_turn": inference_certified_at_turn, + "solved_at_turn": first_eval_pass_turn, + "turns_used": len(turn_records), + "oracle_calls": len(turn_records), + "stagnation_events": stagnation_events, + "refuted_routes": refuted_routes, + "total_prompt_tokens": total_prompt_tokens, + "total_completion_tokens": total_completion_tokens, + "total_tokens": total_prompt_tokens + total_completion_tokens, + "total_retrieved_plane1_tokens": total_retrieved_tokens, + "total_elapsed_ms": round(total_elapsed_ms, 2), + **pass_by_k, + "turns": turn_records, + } + + +def run_repair_split( + items: Sequence[Mapping[str, Any]], + generator_fn: Callable[[str, int], dict[str, Any] | str], + *, + max_turns: int = 4, + retrieval_max_tokens: int = 1250, + inference_oracle_fn: Callable[[Mapping[str, Any], str], Mapping[str, Any]] | None = None, + eval_grader_fn: Callable[[Mapping[str, Any], str], Mapping[str, Any]] | None = None, + grader_fn: Callable[[Mapping[str, Any], str], Mapping[str, Any]] | None = None, + datasets_root: Path | None = None, + spec_path: Path | None = None, + include_plane1_retrieval: bool = True, + checkpoint_jsonl: Path | None = None, +) -> dict[str, Any]: + """Run bounded label-free closed-loop repair across a dataset split with crash-safe checkpointing.""" + completed_by_id: dict[str, dict[str, Any]] = {} + if checkpoint_jsonl is not None and checkpoint_jsonl.is_file(): + for ln in checkpoint_jsonl.read_text(encoding="utf-8").splitlines(): + if ln.strip(): + rec = json.loads(ln) + completed_by_id[str(rec["id"])] = rec + + results: list[dict[str, Any]] = [] + for item in items: + iid = str(item["id"]) + if iid in completed_by_id: + results.append(completed_by_id[iid]) + continue + res = run_repair_item( + item, + generator_fn, + max_turns=max_turns, + retrieval_max_tokens=retrieval_max_tokens, + inference_oracle_fn=inference_oracle_fn, + eval_grader_fn=eval_grader_fn, + grader_fn=grader_fn, + datasets_root=datasets_root, + spec_path=spec_path, + include_plane1_retrieval=include_plane1_retrieval, + ) + results.append(res) + if checkpoint_jsonl is not None: + checkpoint_jsonl.parent.mkdir(parents=True, exist_ok=True) + with checkpoint_jsonl.open("a", encoding="utf-8", newline="\n") as fh: + fh.write(json.dumps(res, sort_keys=True) + "\n") + + n_total = len(results) + pass_curve: dict[str, dict[str, Any]] = {} + for k in range(1, max_turns + 1): + cnt = sum(1 for r in results if r.get(f"pass_at_{k}", False)) + pass_curve[f"k={k}"] = { + "passed": cnt, + "total": n_total, + "pass_rate_bps": int(Fraction(cnt * 10000, n_total)) if n_total else 0, + "pass_rate_pct": round((cnt / n_total) * 100.0, 2) if n_total else 0.0, + } + + by_family: dict[str, dict[str, Any]] = {} + for fam in ("A", "B", "C", "D"): + fam_rows = [r for r in results if r["family"] == fam] + if not fam_rows: + continue + by_family[fam] = { + f"k={k}": { + "passed": sum(1 for r in fam_rows if r.get(f"pass_at_{k}", False)), + "total": len(fam_rows), + } + for k in range(1, max_turns + 1) + } + + return { + "schema": "mcproof-repair-loop-receipt/v0.2", + "label_free_inference_stopping": True, + "max_turns": max_turns, + "retrieval_max_tokens": retrieval_max_tokens, + "include_plane1_retrieval": include_plane1_retrieval, + "total_items": n_total, + "pass_curve": pass_curve, + "by_family": by_family, + "budget_totals": { + "replies_used": sum(int(r["turns_used"]) for r in results), + "oracle_calls": sum(int(r["oracle_calls"]) for r in results), + "stagnation_events": sum(int(r.get("stagnation_events", 0)) for r in results), + "inference_certified_items": sum(1 for r in results if r.get("inference_certified", False)), + "prompt_tokens": sum(int(r["total_prompt_tokens"]) for r in results), + "completion_tokens": sum(int(r["total_completion_tokens"]) for r in results), + "total_tokens": sum(int(r["total_tokens"]) for r in results), + "mean_retrieved_plane1_tokens_per_turn": round( + sum(int(r["total_retrieved_plane1_tokens"]) for r in results) + / max(1, sum(int(r["turns_used"]) for r in results)), + 1, + ), + }, + "items": results, + } diff --git a/stack/results/final/ux582hs_rtx3080_tc5_live_receipt.json b/stack/results/final/ux582hs_rtx3080_tc5_live_receipt.json new file mode 100644 index 0000000..828c659 --- /dev/null +++ b/stack/results/final/ux582hs_rtx3080_tc5_live_receipt.json @@ -0,0 +1,613 @@ +{ + "deviations_and_caveats": [ + "The full `MCPROOF_LIVE_LLAMA_TESTS=1 pytest stack/tests/test_stack_suite.py` end-to-end live test (Phase 4 + full Phase 6 T-C1..T-C5) was NOT run. It needs lab-host prerequisites that the UX582HS host doesn't have (siliconcordat `mde` Rust binary, `exo`, `warp_modeld`, `weft` sibling repos, and the LlamaServerClient they provide); its lab tests self-skip. T-C5 was replayed instead with a standalone harness that reproduces the exact T-C5 server flags, prompt, sampling and interleaving. phase6_receipt.json and docs/CONTINUITY.md in the repo were not modified (no push).", + "llama.cpp build is b12549 (version string 1.0.0-dev); the issue requires >= 0.5.0. The M1 Max reference used llama-server 0.5.0, so absolute tok/s are not comparable across hosts.", + "Measured peak VRAM is well above the analytical baselines in the issue (Q8_0 4.9451 headless / 6.1451 WDDM GiB; Q4_K_M 3.4034 / 4.6034 GiB). Main cause: hardware_probe.compute_rtx3080_vram_fit and the phase6 projection count F16 KV for ONE 8192-token sequence (0.25 GiB), but the T-C5 server config (-np 8, -c 65536, kv_unified=false) allocates KV for 8 x 8192 tokens (about 2.0 GiB), plus 8 recurrent-state copies and the CUDA context. The analytical +1.20 GiB WDDM desktop allowance did not materialize (the dGPU drives no display). Q8_0 headroom under the 7.2 GiB gate is 0.35 GiB.", + "8-slot overhead is an extra measurement (the issue checklist mentions single-slot / 8-slot). The original harness only measures single-slot on the np8 instance. Both are gated here.", + "KV residency stress filled each slot to 7486/8192 tokens (91.4%), not the full 8192; KV is preallocated at boot, so VRAM did not change materially (+10 MiB) under load.", + "A preliminary Q8_0 run (runs/q8_0_prelim.json, before the content-hash LoRA check was added) gave matching numbers: peak 7017 MiB, single-slot overhead -0.22%, 8-slot -1.01%." + ], + "environment": { + "gpu_idle_desktop_vram_mib": 0, + "gpu_nvidia_smi": "NVIDIA GeForce RTX 3080 Laptop GPU, 566.07, 8192 MiB, WDDM, 00000000:01:00.0", + "host": "ASUS ZenBook Pro Duo UX582HS", + "llama_cpp_release": "ggml-org/llama.cpp b12549 (llama-b12549-bin-win-cuda-12.4-x64.zip sha256 05053794f7b5557ac860fbe0572e5b29aee92727f61324899c9641a5d3a6849b + cudart-llama-bin-win-cuda-12.4-x64.zip sha256 8c79a9b226de4b3cacfd1f83d24f962d0773be79f1e7b75c6af4ded7e32ae1d6, both matching GitHub release digests)", + "llama_server_version": [ + "version: 1.0.0-dev (build 12549, commit 8af68d486)", + "built with Clang 22.1.8 for Windows x86_64" + ], + "note_wddm": "dGPU runs under WDDM but drives no display (idle memory.used = 0 MiB); per-process used_memory is N/A under WDDM, so device-wide memory.used is reported.", + "nvidia_smi_cuda_line": "| NVIDIA-SMI 566.07 Driver Version: 566.07 CUDA Version: 12.7 |", + "os": "Windows-11-10.0.26200-SP0", + "python": "3.12.10" + }, + "inputs": { + "gguf_source": "huggingface.co/unsloth/Qwen3.5-4B-GGUF (pinned repo in hardware_probe.py); HF LFS oids equal expected sha256", + "loras": { + "corpus": { + "bytes": 189184, + "llama_adapter_id": 0, + "manifest_sha256": "d81bab6b8c5555674dbbe29e38566435fedc1828d652fbfbf12f35f9562ea640", + "match": true, + "path_in_repo": "compile/artifacts/composition/corpus.f16.gguf", + "sha256": "d81bab6b8c5555674dbbe29e38566435fedc1828d652fbfbf12f35f9562ea640" + }, + "formal-verification": { + "bytes": 189408, + "llama_adapter_id": 2, + "manifest_sha256": "728272c0974b795ceb990fde1e215b62d64e6faf04fb799c0fd3e01fa4539ba5", + "match": true, + "path_in_repo": "compile/artifacts/composition/formal-verification.f16.gguf", + "sha256": "728272c0974b795ceb990fde1e215b62d64e6faf04fb799c0fd3e01fa4539ba5" + }, + "weft-eido": { + "bytes": 189184, + "llama_adapter_id": 1, + "manifest_sha256": "71bedfeee40d0b11a559f73d1c720822a1281452859f43a250bf86bc2755bb86", + "match": true, + "path_in_repo": "compile/artifacts/composition/weft-eido.f16.gguf", + "sha256": "71bedfeee40d0b11a559f73d1c720822a1281452859f43a250bf86bc2755bb86" + } + }, + "mcproof_branch": "feat/formal-verification-mde-v0.1", + "mcproof_commit": "45528746556e4236dcb4a8d10d6fed5b26673c57" + }, + "issue": "Proven-Commons/mcproof#4", + "methodology": { + "eight_slot_protocol": "same prompt and sampling; 8 simultaneous requests (id_slot 0..7, all slots erased first), n_predict 96, 1 warmup per arm (n_predict 64), 5 interleaved trials; tok/s = total predicted tokens / wall-clock", + "gpu_contention_check": "harness refuses to start unless nvidia-smi shows <= 600 MiB used and <= 20% util; both runs started at 0 MiB / 0% with no compute apps", + "harness": "tc5_replay.py (stdlib only), a line-by-line replica of the T-C5 block in stack/src/mcproof_stack/phase6_composition.py (~L2217-2335) and LiveLlamaProcess.build_command in runtime.py", + "kv_gate": "GET /slots: raw_slots[0].n_ctx is int >= 8192 (extract_verified_slot_n_ctx), plus all 8 slots report 8192. Additional stress test: 8 concurrent requests with 7454-token prompts + 32 generated tokens (7486 tokens resident per slot, 91.4% of 8192) under triple LoRA, with peak VRAM sampled during it", + "lora_effect_sanity": "the stack arm's greedy output differs from W0's on every trial (content sha256 recorded), so adapters were actually applied", + "overhead_formula": "(median_w0 - median_stack)/median_w0*100; benchmark_anomaly if < -5.0; gate passes if not anomaly and <= 15.0 (compute_latency_overhead_metrics)", + "server_launch": "llama-server -m --host 127.0.0.1 -c 65536 (=8192*8) -ngl 99 -np 8 --cache-type-k f16 --cache-type-v f16 --slots --slot-save-path --no-cache-idle-slots --slot-prompt-similarity 0.0 --ctx-checkpoints 0 --chat-template-kwargs {\"enable_thinking\":false} --lora-scaled corpus:0.0,weft-eido:0.0,formal-verification:0.0 --lora-init-without-apply; server log confirms n_slots=8, n_ctx_slot=8192, kv_unified=false", + "single_slot_protocol": "POST /completion with BENCH_PROMPT from phase6_composition.py, temperature 0, top_p 1, seed 42, cache_prompt false, id_slot 0. 1 warmup per arm (n_predict 64), then 5 interleaved trials (even: W0 then stack; odd: stack then W0), slot 0 erased before each, n_predict 96. tok/s = timings.predicted_per_second", + "stack_definition": "same server instance, per-request lora=[{id:0,scale:1},{id:1,scale:1},{id:2,scale:1}]", + "vram": "nvidia-smi --query-gpu=memory.used polled about every 100 ms from before server launch until after shutdown (boot, both benchmarks, KV stress); peak = max sample, device-wide (includes CUDA context and cuBLAS workspace)", + "w0_definition": "same server instance, per-request lora=[] (all three adapters loaded at scale 0, not applied)" + }, + "models": { + "Q4_K_M": { + "all_gates_passed": true, + "analytical_projection_from_hardware_probe_py": { + "cuda_context_and_compute_overhead_gib": 0.55, + "deltanet_recurrent_state_gib": 0.0502, + "estimated_total_vram_gib": 3.4034, + "estimated_total_vram_windows_wddm_gib": 4.6034, + "fits_8gib_rtx3080_budget": true, + "fits_8gib_rtx3080_windows_wddm_budget": true, + "kv_cache_8k_f16_gib": 0.25, + "measurement_kind": "analytical_projection_not_live_nvml", + "model_weights_gib": 2.5527, + "rtx3080_budget_gib": 7.2, + "three_f16_rank8_loras_gib": 0.000526, + "windows_wddm_desktop_overhead_gib": 1.2 + }, + "boot_s": 2.29, + "bytes": 2740937888, + "eight_slot_concurrent": { + "anomaly_floor_pct": -5.0, + "benchmark_anomaly": false, + "latency_overhead_pct": 0.48, + "max_allowed_overhead_pct": 15.0, + "metric": "aggregate generated tokens / wall-clock s across 8 concurrent requests", + "overhead_gate_passed": true, + "triple_lora_median_tok_per_s": 344.89, + "triple_lora_samples_tok_per_s": [ + 341.04, + 345.83, + 344.89, + 339.34, + 345.14 + ], + "w0_median_tok_per_s": 346.55, + "w0_samples_tok_per_s": [ + 349.16, + 346.7, + 346.55, + 344.43, + 339.48 + ] + }, + "finished_utc": "2026-11-19T21:34:40.524167+00:00", + "gates": { + "kv_8slot_8192_residency": { + "min_required_n_ctx": 8192, + "passed": true, + "slots_n_ctx": [ + 8192, + 8192, + 8192, + 8192, + 8192, + 8192, + 8192, + 8192 + ], + "stress_all_8_slots_concurrent": true, + "stress_prompt_tokens_per_slot": 7454, + "stress_server_alive_after": true, + "stress_tokens_resident_per_slot_at_end": 7486 + }, + "peak_vram_le_7_2_gib": { + "limit_gib": 7.2, + "measured_peak_gib": 5.2314, + "measured_peak_mib": 5357, + "passed": true + }, + "tok_s_overhead_8slot_le_15pct": { + "benchmark_anomaly": false, + "overhead_pct": 0.48, + "passed": true + }, + "tok_s_overhead_single_slot_le_15pct": { + "benchmark_anomaly": false, + "overhead_pct": 0.47, + "passed": true + } + }, + "gguf": "Qwen3.5-4B-Q4_K_M.gguf", + "kv_residency_stress": { + "aggregate_tok_per_s": 13.252724465069475, + "lora": [ + { + "id": 0, + "scale": 1.0 + }, + { + "id": 1, + "scale": 1.0 + }, + { + "id": 2, + "scale": 1.0 + } + ], + "n_predict": 32, + "per_slot_predicted_per_second": [ + 1.9836884498268241, + 2.2552528658625794, + 2.6114800832944205, + 3.277384156892397, + 4.407019215883238, + 5.998689963514419, + 11.539198844591187, + 44.03390326461677 + ], + "predicted_tokens": 256, + "prompt_n": [ + 7454, + 7454, + 7454, + 7454, + 7454, + 7454, + 7454, + 7454 + ], + "prompt_tokens_per_slot": 7454, + "server_alive_after": true, + "slots_used": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ], + "tokens_resident_per_slot_at_end": 7486, + "wall_s": 19.31678280000051 + }, + "lora_adapters_at_boot": [ + { + "id": 0, + "path": "repo\\Proven-Commons-mcproof-4552874\\compile\\artifacts\\composition\\corpus.f16.gguf", + "prompt_prefix": "", + "scale": 0.0, + "task_name": "" + }, + { + "id": 1, + "path": "repo\\Proven-Commons-mcproof-4552874\\compile\\artifacts\\composition\\weft-eido.f16.gguf", + "prompt_prefix": "", + "scale": 0.0, + "task_name": "" + }, + { + "id": 2, + "path": "repo\\Proven-Commons-mcproof-4552874\\compile\\artifacts\\composition\\formal-verification.f16.gguf", + "prompt_prefix": "", + "scale": 0.0, + "task_name": "" + } + ], + "post_shutdown_nvidia_smi": { + "memory_total_mib": 8192, + "memory_used_mib": 0, + "power_w": "27.52", + "sm_clock_mhz": "210", + "temp_c": "71", + "util_pct": "0" + }, + "pre_run_nvidia_smi": { + "memory_total_mib": 8192, + "memory_used_mib": 0, + "power_w": "23.51", + "sm_clock_mhz": "1245", + "temp_c": "69", + "util_pct": "0" + }, + "raw_run_file": "runs/q4_k_m.json", + "raw_run_file_sha256": "0fbdc15e28943b0cd6a70bb4f661449092dc62ca98fdda68a94a957e9d271e42", + "server_cmd": [ + "llama\\llama-server.exe", + "-m", + "models\\Qwen3.5-4B-Q4_K_M.gguf", + "--host", + "127.0.0.1", + "--port", + "53994", + "-c", + "65536", + "-ngl", + "99", + "-np", + "8", + "--cache-type-k", + "f16", + "--cache-type-v", + "f16", + "--slots", + "--slot-save-path", + "runs\\q4_k_m_slots", + "--no-cache-idle-slots", + "--slot-prompt-similarity", + "0.0", + "--ctx-checkpoints", + "0", + "--chat-template-kwargs", + "{\"enable_thinking\":false}", + "--lora-scaled", + "repo\\Proven-Commons-mcproof-4552874\\compile\\artifacts\\composition\\corpus.f16.gguf:0.0,repo\\Proven-Commons-mcproof-4552874\\compile\\artifacts\\composition\\weft-eido.f16.gguf:0.0,repo\\Proven-Commons-mcproof-4552874\\compile\\artifacts\\composition\\formal-verification.f16.gguf:0.0", + "--lora-init-without-apply" + ], + "sha256_expected": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "sha256_match": true, + "sha256_measured": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "single_slot_tc5_protocol": { + "anomaly_floor_pct": -5.0, + "benchmark_anomaly": false, + "latency_overhead_pct": 0.47, + "lora_effect_check": { + "stack_distinct_outputs": 1, + "stack_output_differs_from_w0": true, + "w0_distinct_outputs": 1 + }, + "max_allowed_overhead_pct": 15.0, + "overhead_gate_passed": true, + "triple_lora_median_tok_per_s": 85.86, + "triple_lora_samples_tok_per_s": [ + 86.04, + 85.91, + 85.86, + 85.42, + 85.62 + ], + "w0_median_tok_per_s": 86.27, + "w0_samples_tok_per_s": [ + 86.53, + 86.4, + 86.27, + 86.2, + 85.91 + ] + }, + "started_utc": "2026-11-19T21:33:33.771074+00:00", + "vram_detail": { + "idle_baseline_mib": 0, + "n_samples": 425, + "peak_minus_baseline_mib": 5357, + "peak_used_gib": 5.2314, + "peak_used_mib": 5357, + "phase_peaks_mib": { + "boot": 5345, + "eight_slot_bench": 5353, + "idle_after": 5357, + "init": 0, + "kv_residency_stress": 5357, + "single_slot_bench": 5351 + }, + "sampling": "nvidia-smi --query-gpu=memory.used every ~100 ms over server lifetime" + } + }, + "Q8_0": { + "all_gates_passed": true, + "analytical_projection_from_hardware_probe_py": { + "cuda_context_and_compute_overhead_gib": 0.55, + "deltanet_recurrent_state_gib": 0.0502, + "estimated_total_vram_gib": 5.0253, + "estimated_total_vram_windows_wddm_gib": 6.2253, + "fits_8gib_rtx3080_budget": true, + "fits_8gib_rtx3080_windows_wddm_budget": true, + "kv_cache_8k_f16_gib": 0.25, + "measurement_kind": "analytical_projection_not_live_nvml", + "model_weights_gib": 4.1746, + "rtx3080_budget_gib": 7.2, + "three_f16_rank8_loras_gib": 0.000526, + "windows_wddm_desktop_overhead_gib": 1.2 + }, + "boot_s": 2.94, + "bytes": 4482403488, + "eight_slot_concurrent": { + "anomaly_floor_pct": -5.0, + "benchmark_anomaly": false, + "latency_overhead_pct": -0.66, + "max_allowed_overhead_pct": 15.0, + "metric": "aggregate generated tokens / wall-clock s across 8 concurrent requests", + "overhead_gate_passed": true, + "triple_lora_median_tok_per_s": 284.65, + "triple_lora_samples_tok_per_s": [ + 285.32, + 283.57, + 284.65, + 282.25, + 285.52 + ], + "w0_median_tok_per_s": 282.78, + "w0_samples_tok_per_s": [ + 282.78, + 275.82, + 287.44, + 286.64, + 282.22 + ] + }, + "finished_utc": "2026-11-19T21:33:28.548342+00:00", + "gates": { + "kv_8slot_8192_residency": { + "min_required_n_ctx": 8192, + "passed": true, + "slots_n_ctx": [ + 8192, + 8192, + 8192, + 8192, + 8192, + 8192, + 8192, + 8192 + ], + "stress_all_8_slots_concurrent": true, + "stress_prompt_tokens_per_slot": 7454, + "stress_server_alive_after": true, + "stress_tokens_resident_per_slot_at_end": 7486 + }, + "peak_vram_le_7_2_gib": { + "limit_gib": 7.2, + "measured_peak_gib": 6.8525, + "measured_peak_mib": 7017, + "passed": true + }, + "tok_s_overhead_8slot_le_15pct": { + "benchmark_anomaly": false, + "overhead_pct": -0.66, + "passed": true + }, + "tok_s_overhead_single_slot_le_15pct": { + "benchmark_anomaly": false, + "overhead_pct": 0.38, + "passed": true + } + }, + "gguf": "Qwen3.5-4B-Q8_0.gguf", + "kv_residency_stress": { + "aggregate_tok_per_s": 13.576640016216954, + "lora": [ + { + "id": 0, + "scale": 1.0 + }, + { + "id": 1, + "scale": 1.0 + }, + { + "id": 2, + "scale": 1.0 + } + ], + "n_predict": 32, + "per_slot_predicted_per_second": [ + 1.968550563205491, + 2.3201651239065475, + 2.6777080373998072, + 3.3477357966909254, + 4.475282509426605, + 6.044151552225856, + 11.263311144610379, + 36.77024409511395 + ], + "predicted_tokens": 256, + "prompt_n": [ + 7454, + 7454, + 7454, + 7454, + 7454, + 7454, + 7454, + 7454 + ], + "prompt_tokens_per_slot": 7454, + "server_alive_after": true, + "slots_used": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ], + "tokens_resident_per_slot_at_end": 7486, + "wall_s": 18.855917200000476 + }, + "lora_adapters_at_boot": [ + { + "id": 0, + "path": "repo\\Proven-Commons-mcproof-4552874\\compile\\artifacts\\composition\\corpus.f16.gguf", + "prompt_prefix": "", + "scale": 0.0, + "task_name": "" + }, + { + "id": 1, + "path": "repo\\Proven-Commons-mcproof-4552874\\compile\\artifacts\\composition\\weft-eido.f16.gguf", + "prompt_prefix": "", + "scale": 0.0, + "task_name": "" + }, + { + "id": 2, + "path": "repo\\Proven-Commons-mcproof-4552874\\compile\\artifacts\\composition\\formal-verification.f16.gguf", + "prompt_prefix": "", + "scale": 0.0, + "task_name": "" + } + ], + "post_shutdown_nvidia_smi": { + "memory_total_mib": 8192, + "memory_used_mib": 0, + "power_w": "13.40", + "sm_clock_mhz": "210", + "temp_c": "72", + "util_pct": "0" + }, + "pre_run_nvidia_smi": { + "memory_total_mib": 8192, + "memory_used_mib": 0, + "power_w": "19.78", + "sm_clock_mhz": "1245", + "temp_c": "61", + "util_pct": "0" + }, + "raw_run_file": "runs/q8_0.json", + "raw_run_file_sha256": "96920b45e86d4f611edbe821b666de43fb38ef08878c8e323250812bcf4c5b94", + "server_cmd": [ + "llama\\llama-server.exe", + "-m", + "models\\Qwen3.5-4B-Q8_0.gguf", + "--host", + "127.0.0.1", + "--port", + "51309", + "-c", + "65536", + "-ngl", + "99", + "-np", + "8", + "--cache-type-k", + "f16", + "--cache-type-v", + "f16", + "--slots", + "--slot-save-path", + "runs\\q8_0_slots", + "--no-cache-idle-slots", + "--slot-prompt-similarity", + "0.0", + "--ctx-checkpoints", + "0", + "--chat-template-kwargs", + "{\"enable_thinking\":false}", + "--lora-scaled", + "repo\\Proven-Commons-mcproof-4552874\\compile\\artifacts\\composition\\corpus.f16.gguf:0.0,repo\\Proven-Commons-mcproof-4552874\\compile\\artifacts\\composition\\weft-eido.f16.gguf:0.0,repo\\Proven-Commons-mcproof-4552874\\compile\\artifacts\\composition\\formal-verification.f16.gguf:0.0", + "--lora-init-without-apply" + ], + "sha256_expected": "10cc391b403021dd11c614679d2fd92f611c3681d29e29651b717316965d61e1", + "sha256_match": true, + "sha256_measured": "10cc391b403021dd11c614679d2fd92f611c3681d29e29651b717316965d61e1", + "single_slot_tc5_protocol": { + "anomaly_floor_pct": -5.0, + "benchmark_anomaly": false, + "latency_overhead_pct": 0.38, + "lora_effect_check": { + "stack_distinct_outputs": 1, + "stack_output_differs_from_w0": true, + "w0_distinct_outputs": 1 + }, + "max_allowed_overhead_pct": 15.0, + "overhead_gate_passed": true, + "triple_lora_median_tok_per_s": 63.91, + "triple_lora_samples_tok_per_s": [ + 63.89, + 64.18, + 63.96, + 63.88, + 63.91 + ], + "w0_median_tok_per_s": 64.16, + "w0_samples_tok_per_s": [ + 64.48, + 63.76, + 64.33, + 64.1, + 64.16 + ] + }, + "started_utc": "2026-11-19T21:32:12.181135+00:00", + "vram_detail": { + "idle_baseline_mib": 0, + "n_samples": 492, + "peak_minus_baseline_mib": 7017, + "peak_used_gib": 6.8525, + "peak_used_mib": 7017, + "phase_peaks_mib": { + "boot": 7007, + "eight_slot_bench": 7013, + "idle_after": 7017, + "init": 0, + "kv_residency_stress": 7017, + "single_slot_bench": 7013 + }, + "sampling": "nvidia-smi --query-gpu=memory.used every ~100 ms over server lifetime" + } + } + }, + "overall_passed": true, + "receipt": "ux582hs-tc5-rtx3080-live-replay", + "repo_unit_tests": { + "command": "python -m unittest tests.test_stack_suite -v (stack/, PYTHONPATH=src)", + "log": "stack_unittest.log", + "result": "13 run: 10 passed, 0 failed, 3 skipped" + }, + "summary": { + "Q4_K_M": { + "eight_slot_overhead_pct": 0.48, + "kv_8slot_8192_residency": true, + "peak_vram_gib": 5.2314, + "peak_vram_le_7_2_gib": true, + "single_slot_overhead_pct": 0.47, + "stack_single_slot_tok_s": 85.86, + "tok_s_overhead_8slot_le_15pct": true, + "tok_s_overhead_single_slot_le_15pct": true, + "w0_single_slot_tok_s": 86.27 + }, + "Q8_0": { + "eight_slot_overhead_pct": -0.66, + "kv_8slot_8192_residency": true, + "peak_vram_gib": 6.8525, + "peak_vram_le_7_2_gib": true, + "single_slot_overhead_pct": 0.38, + "stack_single_slot_tok_s": 63.91, + "tok_s_overhead_8slot_le_15pct": true, + "tok_s_overhead_single_slot_le_15pct": true, + "w0_single_slot_tok_s": 64.16 + } + }, + "task": "Stage 1 Task 1: live T-C5 serving-budget replay on 8 GiB RTX 3080 Laptop GPU" +} diff --git a/tests/test_domain_and_repair_loop.py b/tests/test_domain_and_repair_loop.py new file mode 100644 index 0000000..190e167 --- /dev/null +++ b/tests/test_domain_and_repair_loop.py @@ -0,0 +1,459 @@ +"""Tests for domain-general DomainOracle protocol, label-free repair loop, stagnation escape, sandbox, and F-01..F-08/A6/A7 gates.""" + +from __future__ import annotations + +import io +import json +from pathlib import Path +import sys +from typing import Any, Mapping, Sequence + +from mcproof.cli import main as cli_main +from mcproof.domain_protocol import ( + BudgetLedger, + DomainDiagnostic, + DomainGradeResult, + DomainOracle, + FormalVerificationDomainOracle, + audit_portfolio_bounds, + audit_transition_certificate, + loads_strict_certificate_json, + run_restricted_sandbox_cell, + verify_dual_product_parity, + verify_minimal_goal_stopping, + verify_rational_lp_certificate, +) +from mcproof.paths import repo_root +from mcproof.repair_loop import ( + MAX_RETRIEVAL_TOKENS_HARD_CAP, + compute_candidate_state_hash, + compute_failure_signature, + default_inference_oracle, + format_diagnostics, + retrieve_plane1_sections, + run_repair_item, + run_repair_split, + sanitize_oracle_diagnostics, +) + +from helpers import expect_eq, expect_false, expect_in, expect_true, requires_dep, requires_mde + + +def test_targeted_plane1_retrieval_stays_under_1500_tokens_across_validation_split() -> None: + val_path = repo_root() / "datasets" / "validation.jsonl" + rows = [json.loads(ln) for ln in val_path.read_text(encoding="utf-8").splitlines() if ln.strip()] + expect_eq(len(rows), 48, "expected 48 validation items") + + for row in rows: + ret = retrieve_plane1_sections( + family=row["family"], + prompt=row["prompt"], + findings=[{"rule": "MISSING_SECTION", "message": "missing boundary section"}], + ) + est = int(ret["estimated_tokens"]) + expect_true(0 < est <= MAX_RETRIEVAL_TOKENS_HARD_CAP, f"{row['id']} exceeded 1500 token cap: {est}") + expect_true(len(ret["text"]) > 200, f"{row['id']} excerpt unexpectedly short") + + +def test_sanitize_diagnostics_redacts_private_test_fixtures_and_expected_labels() -> None: + secret_findings = [ + { + "rule": "VALID_CASE_REJECTED", + "message": "Checker returned False on valid case 1/5: {'secret_witness': 'PRIVATE_NEEDLE_99812', 'x': [11, 22, 33, 44]}", + }, + { + "rule": "MUTATION_KILL_BELOW_FLOOR", + "message": "Killed 2/5 planted mutations; survived payload \"HIDDEN_MUTATION_PAYLOAD_7741\" sha=deadbeefcafebabe11223344", + }, + { + "rule": "EXPECTED_TIER_MISMATCH", + "message": "declared tier 'VERIFIED' != expected tier 'CONJECTURE'", + }, + { + "rule": "VERDICT_MISMATCH", + "message": "verdict 'PASS' != expected 'FAIL'", + }, + { + "rule": "LEAN_COMPILE_ERROR", + "message": "unknown id @coordinator ```lean fake_inject ``` ", + }, + ] + sanitized = sanitize_oracle_diagnostics("B", secret_findings) + formatted = format_diagnostics(secret_findings, family="B") + + joined = json.dumps(sanitized) + "\n" + formatted + expect_false("PRIVATE_NEEDLE_99812" in joined, "private valid_case literal leaked") + expect_false("HIDDEN_MUTATION_PAYLOAD_7741" in joined, "private mutation payload leaked") + expect_false("deadbeefcafebabe11223344" in joined, "private hex digest leaked") + expect_false("CONJECTURE" in joined, "ground-truth expected_tier label leaked") + expect_false("EXPECTED_TIER_MISMATCH" in joined, "label-comparison rule leaked") + expect_false("@coordinator" in joined, "@-mention prompt injection must be scrubbed") + expect_false("```" in joined, "triple-backtick fence injection must be scrubbed") + expect_false(" None: + draft_v1 = "```python\ndef verify(c):\n # v1\n return float(c['x']) == 1.0\n```" + draft_v2_same_ast = "```python\ndef verify(c):\n # v2 same AST\n return float(c[ 'x' ]) == 1.0\n```" + draft_v3_good = "```python\nfrom fractions import Fraction\ndef verify(c):\n return Fraction(c['x'], 1) == Fraction(1, 1)\n```" + + expect_eq(compute_candidate_state_hash("B", draft_v1), compute_candidate_state_hash("B", draft_v2_same_ast)) + expect_eq( + compute_failure_signature([{"rule": "FLOAT_CALL", "line": 3}]), + compute_failure_signature([{"rule": "FLOAT_CALL", "line": 3, "message": "other"}]), + ) + + task = { + "id": "PURE-B-01", + "family": "B", + "expected_tier": "SECRET_DO_NOT_LEAK", + "prompt": "Write exact checker `def verify(c):` testing `x == 1`.", + } + + def _pure_inf_oracle(pub: Mapping[str, Any], comp: str) -> dict[str, Any]: + expect_eq(set(pub.keys()), {"id", "family", "prompt"}) + has_float = "float(" in comp + return { + "passed": not has_float, + "epistemic_status": "LINT_PASS" if not has_float else "REJECTED", + "findings": [{"rule": "FLOAT_COERCION_CALL", "line": 3, "message": "float() forbidden"}] if has_float else [], + } + + def _posthoc_eval(_item: Mapping[str, Any], comp: str) -> dict[str, Any]: + ok = "Fraction(" in comp and "float(" not in comp + return { + "passed": ok, + "findings": [] if ok else [{"rule": "SECRET_EVAL_CANARY", "message": "HIDDEN_GROUND_TRUTH_991"}], + } + + prompts: list[str] = [] + + def _gen(up: str, t_idx: int) -> str: + prompts.append(up) + expect_false("SECRET_DO_NOT_LEAK" in up or "HIDDEN_GROUND_TRUTH_991" in up) + return draft_v1 if t_idx == 1 else (draft_v2_same_ast if t_idx == 2 else draft_v3_good) + + ckpt = tmp_path / "pure_ckpt.jsonl" + split = run_repair_split( + [task], + _gen, + max_turns=4, + inference_oracle_fn=_pure_inf_oracle, + eval_grader_fn=_posthoc_eval, + checkpoint_jsonl=ckpt, + ) + res = split["items"][0] + expect_true(res["passed"]) + expect_eq(res["turns_used"], 3) + expect_true(res["stagnation_events"] >= 1) + expect_in("Stagnation Escape Alert", prompts[2]) + expect_in("Refuted Routes (Do Not Re-Explore)", prompts[2]) + expect_eq(len(res["refuted_routes"]), 2) + + prompts.clear() + split_resumed = run_repair_split( + [task], + _gen, + max_turns=4, + inference_oracle_fn=_pure_inf_oracle, + eval_grader_fn=_posthoc_eval, + checkpoint_jsonl=ckpt, + ) + expect_eq(len(prompts), 0, "checkpoint resume must skip completed item") + expect_eq(split_resumed["pass_curve"]["k=4"]["passed"], 1) + + +@requires_dep("soko") +@requires_mde() +def test_label_free_default_inference_oracle_with_live_soko_and_mde() -> None: + good_checker = """```python +from fractions import Fraction + +def verify(case: dict) -> bool: + if not isinstance(case, dict): + return False + a = case.get("a") + b = case.get("b") + s = case.get("sum") + if any(isinstance(v, bool) or not isinstance(v, int) for v in (a, b, s)): + return False + return Fraction(a, 1) + Fraction(b, 1) == Fraction(s, 1) +```""" + bad_checker = """```python +def verify(case: dict) -> bool: + return float(case["a"]) + float(case["b"]) == float(case["sum"]) +```""" + task = { + "id": "SYNTH-B-01", + "family": "B", + "expected_tier": "SECRET_GROUND_TRUTH_TIER_DO_NOT_READ", + "valid_cases": [{"a": 999111, "b": 222333, "sum": 1221444}], + "prompt": "Write an exact-integer sum verifier `def verify(case: dict) -> bool:` checking `a + b == sum`.", + } + inf_bad = default_inference_oracle(task, bad_checker) + expect_false(inf_bad["passed"], "float() checker must fail static inference audit") + expect_false("SECRET_GROUND_TRUTH_TIER_DO_NOT_READ" in json.dumps(inf_bad)) + expect_false("999111" in json.dumps(inf_bad)) + + inf_good = default_inference_oracle(task, good_checker) + expect_true(inf_good["passed"], f"exact Fraction checker must pass inference audit: {inf_good}") + expect_eq(inf_good["epistemic_status"], "LINT_PASS") + + +def test_pluggable_domain_oracle_protocol_and_budget_ledger() -> None: + builtin = FormalVerificationDomainOracle() + expect_true(isinstance(builtin, DomainOracle), "FormalVerificationDomainOracle must satisfy DomainOracle Protocol") + ctx = builtin.retrieve_plane1_context({"family": "A", "prompt": "Prove Nat clamping bounds using omega."}) + expect_true("Lean" in ctx, "expected Lean Plane 1 guidance") + + class ToyGraphOracle: + @property + def domain_id(self) -> str: + return "mde:toy-graph" + + def retrieve_plane1_context( + self, + task: Mapping[str, Any], + prior_diagnostics: Sequence[DomainDiagnostic] = (), + max_tokens: int = 1250, + ) -> str: + return f"Graph spec for {task['id']} (max={max_tokens}, errs={len(prior_diagnostics)})" + + def grade_candidate(self, task: Mapping[str, Any], candidate: str) -> DomainGradeResult: + ok = candidate.strip() == str(task["expected"]) + diags = () if ok else (DomainDiagnostic(rule="WRONG_PATH_COST", message="path cost mismatch"),) + return DomainGradeResult(passed=ok, epistemic_status="VERIFIED" if ok else "REJECTED", diagnostics=diags, metrics={}) + + def format_repair_feedback(self, _task: Mapping[str, Any], grade: DomainGradeResult) -> str: + return "; ".join(f"{d.rule}:{d.message}" for d in grade.diagnostics) + + ext = ToyGraphOracle() + expect_true(isinstance(ext, DomainOracle), "external specialist class must satisfy DomainOracle without inheritance") + g_fail = ext.grade_candidate({"id": "G1", "expected": "4"}, "5") + expect_false(g_fail.passed) + expect_in("WRONG_PATH_COST", ext.format_repair_feedback({"id": "G1"}, g_fail)) + + ledger = BudgetLedger(max_turns=4, max_tokens=1000, max_steps=50) + ledger.turns_used = 2 + expect_false(ledger.exhausted()) + ledger.steps_used = 50 + expect_true(ledger.exhausted()) + + +def test_restricted_sandbox_cell_enforces_imports_builtins_and_timeout() -> None: + ok_res = run_restricted_sandbox_cell( + "from collections import deque\nfrom fractions import Fraction\nq = deque([Fraction(1, 3), Fraction(2, 3)])\nprint(sum(q) + OFFSET)\n", + injected_globals={"OFFSET": 4}, + ) + expect_true(ok_res["ok"], f"valid pure-algorithmic cell rejected: {ok_res}") + expect_eq(ok_res["stdout"].strip(), "5") + + for bad_code in ( + "import socket\n", + "import subprocess\n", + "from urllib import request\n", + "import ext_helper\n", + "f = open('/etc/hosts')\n", + "x = ().__class__.__bases__[0].__subclasses__()\n", + ): + rej = run_restricted_sandbox_cell(bad_code, extra_forbidden_modules=["ext_helper"]) + expect_false(rej["ok"], f"forbidden cell should be rejected: {bad_code!r}") + expect_true(rej["ast_rejected"], f"expected AST rejection on {bad_code!r}") + + t_out = run_restricted_sandbox_cell("while True:\n x = 1\n", timeout_sec=0.4) + expect_false(t_out["ok"], "infinite loop must fail") + expect_true(t_out["timed_out"], "infinite loop must set timed_out=True") + + +def test_dual_product_parity_portfolio_bounds_and_goal_stopping_close_f06_f07_and_overrun() -> None: + f06_coarse = verify_dual_product_parity( + spec_init=(0, 0), + impl_init=(0, 0), + spec_actions=lambda s: ("tick",) if s[0] < 2 else (), + impl_actions=lambda u: ("tick",) if u[0] < 2 else (), + spec_step=lambda s, _a: (s[0] + 1, 0), + impl_step=lambda u, _a: (u[0] + 1, 1 if u[0] + 1 == 2 else 0), + spec_invariant=lambda s: s[1] == 0, + impl_invariant=lambda u: u[1] == 0, + spec_is_goal=lambda s: s[0] == 2, + impl_is_goal=lambda u: u[0] == 2, + spec_obs=lambda _s: 0, + impl_obs=lambda _u: 0, + ) + expect_false(f06_coarse["passed"]) + expect_in("F06_IMPL_INVARIANT_VIOLATION_OR_DIVERGENCE", {f["rule"] for f in f06_coarse["findings"]}) + + f06_collapsed = verify_dual_product_parity( + spec_init=0, + impl_init=0, + spec_actions=lambda s: ("a", "b") if s == 0 else (), + impl_actions=lambda u: ("a", "b") if u == 0 else (), + spec_step=lambda _s, _act: 1, + impl_step=lambda _u, act: 1 if act == "a" else 2, + spec_invariant=lambda s: s in (0, 1), + impl_invariant=lambda u: u in (0, 1), + spec_is_goal=lambda s: s == 1, + impl_is_goal=lambda u: u in (1, 2), + spec_obs=lambda s: 0 if s == 0 else 1, + impl_obs=lambda u: 0 if u == 0 else 1, + ) + expect_false(f06_collapsed["passed"]) + expect_in("F06_IMPL_INVARIANT_VIOLATION_OR_DIVERGENCE", {f["rule"] for f in f06_collapsed["findings"]}) + + f06_clean = verify_dual_product_parity( + spec_init=0, + impl_init=0, + spec_actions=lambda s: ("a", "b") if s < 2 else (), + impl_actions=lambda u: ("a", "b") if u < 2 else (), + spec_step=lambda s, act: s + (1 if act == "a" else 2), + impl_step=lambda u, act: u + (1 if act == "a" else 2), + spec_invariant=lambda s: 0 <= s <= 3, + impl_invariant=lambda u: 0 <= u <= 3, + spec_is_goal=lambda s: s == 2, + impl_is_goal=lambda u: u == 2, + spec_obs=lambda s: s, + impl_obs=lambda u: u, + ) + expect_true(f06_clean["passed"]) + expect_eq(f06_clean["epistemic_status"], "PROVEN") + + port_bad = audit_portfolio_bounds( + [ + {"id": "ITEM-INV", "proven_floor": 50, "shortest_known": 48, "epistemic_status": "VERIFIED"}, + {"id": "ITEM-UNREG", "proven_floor": 40, "shortest_known": 48, "epistemic_status": "VERIFIED"}, + ], + registered_verified_ids=frozenset(), + ) + expect_false(port_bad["passed"]) + p_rules = {f["rule"] for f in port_bad["findings"]} + expect_in("CERT_F07_INCONSISTENT_PORTFOLIO_BOUNDS", p_rules) + expect_in("CERT_F07_UNVERIFIED_CALLER_BOUNDS", p_rules) + + overrun = verify_minimal_goal_stopping( + init_state=0, + actions=[1, 1, 1], + step_fn=lambda s, a: s + a, + is_goal_fn=lambda s: s == 2, + ) + expect_false(overrun["passed"]) + expect_in("CERT_POST_GOAL_EXTRA_ACTIONS", {f["rule"] for f in overrun["findings"]}) + + exact_stop = verify_minimal_goal_stopping( + init_state=0, + actions=[1, 1], + step_fn=lambda s, a: s + a, + is_goal_fn=lambda s: s == 2, + ) + expect_true(exact_stop["passed"]) + + +def test_audit_transition_certificate_catches_f01_to_f08_and_a6_a7(tmp_path: Path) -> None: + valid_cert = { + "epistemic_status": "VERIFIED", + "proof_mode": "quotient_homomorphism", + "complete_space": True, + "complete_domain_closed": True, + "differential_match": True, + "oracles_checked": ["ref_spec_oracle", "flat_bitmask_oracle"], + "includes_hidden_state": True, + "exact_optimum_reached": True, + "proven_floor": 12, + "witness": "UUDDLRLRBAAB", + "witness_cost": 12, + "witness_or_proof_sha256": "a" * 64, + "instance_sha256": "b" * 64, + "canonical_instance_sha256": "b" * 64, + "step_consistency_violations": 0, + "must_fail_controls": ["corrupt_floor_plus_one", "truncate_witness_tail"], + "control_records": [ + {"name": "corrupt_floor_plus_one", "oracle_invoked": True, "unexpectedly_passed": False}, + {"name": "truncate_witness_tail", "oracle_invoked": True, "unexpectedly_passed": False}, + ], + "tcb_pins": { + "tcb_manifest_verified": True, + "checker_sha256": "c" * 64, + }, + } + clean = audit_transition_certificate( + valid_cert, + require_must_fail_controls=True, + registered_artifact_digests=frozenset({"a" * 64}), + ) + expect_true(clean["passed"], f"valid certificate rejected: {clean}") + + corruptions: list[tuple[str, dict[str, Any]]] = [ + ("CERT_SINGLE_ORACLE_DIFFERENTIAL", {**valid_cert, "oracles_checked": ["only_one"]}), + ("CERT_PARTIAL_STATE_PROJECTION", {**valid_cert, "includes_hidden_state": False}), + ("CERT_TCB_MANIFEST_UNLOCKED", {**valid_cert, "tcb_pins": {"tcb_manifest_verified": False, "checker_sha256": "short"}}), + ("CERT_UNWITNESSED_VERIFIED_OPTIMUM", {**valid_cert, "witness": None, "exact_optimum_reached": False}), + ("CERT_STEP_INCONSISTENCY", {**valid_cert, "step_consistency_violations": 1}), + ("CERT_TRUNCATED_INDUCTIVE_POTENTIAL", {**valid_cert, "proof_mode": "inductive_potential", "complete_domain_closed": False}), + ( + "CERT_IMPL_INVARIANT_SKIPPED", + {**valid_cert, "epistemic_status": "PROVEN", "impl_invariant_checked": False, "obs_projection_injective": False}, + ), + ("CERT_UNVERIFIED_CALLER_INFLATION", {**valid_cert, "caller_bounds_verified": False}), + ("CERT_INVALID_WITNESS_SHA256", {**valid_cert, "witness_or_proof_sha256": "ab"}), + ("CERT_A6_MISSING_OR_DUPLICATE_CONTROLS", {**valid_cert, "must_fail_controls": ["dup_ctrl", "dup_ctrl"]}), + ( + "CERT_A6_TAUTOLOGICAL_CONTROL", + { + **valid_cert, + "control_records": [{"name": "seeded_noop", "oracle_invoked": False, "seed_equals_target": True}], + }, + ), + ("CERT_A7_ILLEGAL_TIER_SWAP", {**valid_cert, "prior_status": "LINT_PASS"}), + ("CERT_A7_ILLEGAL_TIER_SWAP", {**valid_cert, "prior_status": "LOWER_BOUND_UNWITNESSED"}), + ] + for expected_rule, bad_cert in corruptions: + res = audit_transition_certificate(bad_cert, require_must_fail_controls=True) + expect_false(res["passed"], f"expected {expected_rule} to fail closed") + rules = {f["rule"] for f in res["findings"]} + expect_in(expected_rule, rules) + + unreg = audit_transition_certificate( + valid_cert, + registered_artifact_digests=frozenset({"d" * 64}), + ) + expect_false(unreg["passed"]) + expect_in("CERT_UNREGISTERED_WITNESS_DIGEST", {f["rule"] for f in unreg["findings"]}) + + cert_file = tmp_path / "cert.json" + cert_file.write_text(json.dumps(valid_cert), encoding="utf-8", newline="\n") + old_stdout = sys.stdout + sys.stdout = io.StringIO() + try: + expect_eq(cli_main(["--json", "audit-cert", str(cert_file)]), 0) + for bad_raw in ( + '{"proven_floor": 12.5}', + '{"proven_floor": 12, "proven_floor": 13}', + '{"proven_floor": ' + ("9" * 65) + "}", + ): + bad_file = tmp_path / "bad_strict.json" + bad_file.write_text(bad_raw, encoding="utf-8", newline="\n") + expect_false(cli_main(["--json", "audit-cert", str(bad_file)]) == 0) + finally: + sys.stdout = old_stdout + expect_eq(loads_strict_certificate_json('{"a": 42, "b": "7/3"}')["a"], 42) + + lp_bounds_ok = verify_rational_lp_certificate( + { + "mode": "bounds", + "A": [[1, 1], [2, 0]], + "b": [9, 8], + "c": [2, 1], + "y": ["1", "1/2"], + "claimed_lower_bound": "13", + } + ) + expect_true(lp_bounds_ok["passed"]) + expect_eq(lp_bounds_ok["dual_objective"], "13/1") + + lp_bool_sneak = verify_rational_lp_certificate( + {"mode": "bounds", "A": [[1]], "b": [True], "c": [1], "y": [1], "claimed_lower_bound": 1} + ) + expect_false(lp_bool_sneak["passed"]) + + lp_farkas_ok = verify_rational_lp_certificate({"mode": "infeasible", "A": [[-1, -1]], "b": [1], "y": [1]}) + expect_true(lp_farkas_ok["passed"]) diff --git a/tests/test_lean_harness.py b/tests/test_lean_harness.py index 7a56c2b..b823627 100644 --- a/tests/test_lean_harness.py +++ b/tests/test_lean_harness.py @@ -10,13 +10,17 @@ from mcproof.lean_harness import ( BAR_FORBIDDEN_RULES, - _lean_env, + _lean_env, audit_weak_statements, + classify_host_resource_failure, lint_lean4_source, parse_comparator_stdout, + scrub_challenge_artifacts, verify_lean4_project_comparator, verify_lean4_source, + verify_pinned_lean_defs, ) +from mcproof.oracles import run_lean4_oracle from mcproof.paths import repo_root from helpers import expect_eq, expect_false, expect_in, expect_true, requires_dep, requires_lean @@ -124,7 +128,7 @@ def test_reference_lean_project_passes_six_kernel_comparator() -> None: res = verify_lean4_project_comparator(repo_root() / "lean", allow_unsandboxed=not has_bwrap) expect_true(res.passed, f"reference lean project failed: {res.findings}") expect_eq(len(res.paranoid_kernels_run), 6, "all 6 bundled Lean 4.35.0 kernels must accept") - expect_eq(len(res.theorems_checked), 8, "all 8 theorems in comparator.json must be checked") + expect_eq(len(res.theorems_checked), 29, "all 29 theorems in comparator.json must be checked") expect_eq(len(res.sorry_declarations), 0, "zero sorry declarations allowed in Mcproof.Proofs") expect_eq(len(res.nonstandard_axioms), 0, "zero non-standard axioms allowed") expect_eq(res.challenge_sorry_whitelisted_module, "Mcproof.Spec") @@ -256,3 +260,54 @@ def test_rejects_weak_statement_vacuous_hypotheses() -> None: res3 = lint_lean4_source(contradictory) expect_false(res3["passed"], "contradictory P and Not P binders must fail") expect_in("VACUOUS_CONTRADICTORY_HYPOTHESES", {f["rule"] for f in res3["findings"]}) + + taut_hyp = """ +theorem circular_gap (gap : Int) (h : 0 < gap) : 0 < gap := h +""" + res4 = lint_lean4_source(taut_hyp) + expect_false(res4["passed"], "circular binder premise matching conclusion must fail") + expect_in("VACUOUS_TAUTOLOGICAL_HYPOTHESIS", {f["rule"] for f in res4["findings"]}) + + +def test_pinned_def_mutation_challenge_scrub_and_host_limit_pure(tmp_path: Path) -> None: + prompt_lean = """ +def targetClamp (x : Nat) : Nat := + if x > 10 then 10 else x +""" + forged_cand = """ +def targetClamp (_x : Nat) : Nat := 10 + +theorem targetClamp_le_ten (x : Nat) : targetClamp x ≤ 10 := by + omega +""" + honest_cand = """ +def targetClamp (x : Nat) : Nat := + if x > 10 then 10 else x + +theorem targetClamp_le_ten (x : Nat) : targetClamp x ≤ 10 := by + unfold targetClamp + split <;> omega +""" + expect_in("TARGET_DEF_BODY_MUTATED", {f["rule"] for f in verify_pinned_lean_defs(prompt_lean, forged_cand)}) + expect_eq(verify_pinned_lean_defs(prompt_lean, honest_cand), []) + + expect_true(classify_host_resource_failure(137, "") is not None) + expect_true(classify_host_resource_failure(1, "error: excessive memory consumption") is not None) + expect_true(classify_host_resource_failure(1, "type mismatch at application") is None) + + fake_proj = tmp_path / "proj" + olean_dir = fake_proj / ".lake" / "build" / "lib" / "lean" / "Mcproof" + olean_dir.mkdir(parents=True) + stale_olean = olean_dir / "Spec.olean" + stale_olean.write_bytes(b"stale") + removed = scrub_challenge_artifacts(fake_proj, "Mcproof.Spec") + expect_false(stale_olean.exists(), "Spec.olean must be unlinked") + expect_eq(removed, [".lake/build/lib/lean/Mcproof/Spec.olean"]) + + +@requires_dep("soko") +@requires_lean() +def test_lean_timeout_maps_to_incomplete_host_limit() -> None: + t_rec = run_lean4_oracle("theorem quick_ok : 1 + 1 = 2 := rfl", timeout_sec=0) + expect_false(t_rec.passed, "zero-second timeout must fail") + expect_eq(t_rec.epistemic_status, "INCOMPLETE_HOST_LIMIT", "timeout must map to INCOMPLETE_HOST_LIMIT") diff --git a/tests/test_lineage_prose_and_cli.py b/tests/test_lineage_prose_and_cli.py index 75d9571..e04e392 100644 --- a/tests/test_lineage_prose_and_cli.py +++ b/tests/test_lineage_prose_and_cli.py @@ -162,6 +162,7 @@ def test_vendored_prose_gate_runs_without_home_directory_scripts( md_files = [ r_root / "README.md", r_root / "docs" / "CONTINUITY.md", + r_root / "docs" / "CROSS_POLLINATION.md", r_root / "spec" / "plane1_formal_spec.md", r_root / "lean" / "README.md", r_root / "lean" / "SPEC.md", diff --git a/tests/test_neon_audit_regressions.py b/tests/test_neon_audit_regressions.py new file mode 100644 index 0000000..9136700 --- /dev/null +++ b/tests/test_neon_audit_regressions.py @@ -0,0 +1,285 @@ +"""Cross-architecture adversarial regression suite (`NEON-C1..C4` & `NEON-D1/D2/D5`).""" + +from __future__ import annotations + +import json +from pathlib import Path +import tempfile + +from mcproof.domain_protocol import ( + audit_sandbox_cell_ast, + audit_transition_certificate, + run_restricted_sandbox_cell, +) +from mcproof.lean_harness import ( + audit_weak_statements, + build_capped_lean_args, + extract_normalized_lean_defs, + verify_pinned_lean_defs, +) +from mcproof.mutation import audit_exact_checker_ast, run_checker_mutation_suite +from mcproof.oracles import run_checker_oracle +from mcproof.paths import repo_root +from mcproof.repair_loop import ( + _strip_lean_comments, + compute_candidate_state_hash, + default_dataset_item_grader, + default_inference_oracle, + retrieve_plane1_excerpt, + run_repair_item, +) + +from helpers import expect_eq, expect_false, expect_in, expect_true, requires_dep, requires_lean, requires_mde + + +def test_neon_c1_pure_ast_allowlist_and_mutation_suite_confinement(tmp_path: Path) -> None: + marker = tmp_path / "MUST_NEVER_BE_WRITTEN.txt" + malicious_py = ( + "import pathlib\n" + f"pathlib.Path({str(marker)!r}).write_text('PWN', encoding='utf-8')\n" + "def verify(x):\n" + " return x == 1\n" + ) + ast_findings = audit_exact_checker_ast(malicious_py) + expect_in("FORBIDDEN_CHECKER_IMPORT", {f["rule"] for f in ast_findings}) + + mut_rep = run_checker_mutation_suite( + malicious_py, + valid_cases=[{"args": [1]}], + planted_mutations=[{"name": "m0", "args": [2]}], + ) + expect_false(mut_rep.passed) + expect_false(marker.exists(), "mutation suite must never execute unallowlisted imports on host") + + neg_pow_py = "def verify(x: int) -> bool:\n return (2 ** -1) == x\n" + expect_in("NEGATIVE_POW_FLOAT_COERCION", {f["rule"] for f in audit_exact_checker_ast(neg_pow_py)}) + + for hop_py in ( + "import typing\n_io = typing.sys.modules.get('_io')\ndef verify(x): return x == 1\n", + "import collections\ns = collections._sys\ndef verify(x): return x == 1\n", + "from json import codecs\ndef verify(x): return x == 1\n", + "from fractions import *\ndef verify(x): return x == 1\n", + "from collections import _sys\ndef verify(x): return x == 1\n", + "fn = open\ndef verify(x): return x == 1\n", + ): + expect_true(len(audit_exact_checker_ast(hop_py)) > 0, f"module-hop/builtin bypass missed: {hop_py!r}") + + +@requires_dep("soko") +@requires_mde() +def test_neon_c1_run_checker_oracle_never_executes_unallowlisted_or_rejected_code(tmp_path: Path) -> None: + marker_pass = tmp_path / "HOST_EXECUTED_AND_PASSED.txt" + malicious_pass_py = ( + "import pathlib\n" + f"pathlib.Path({str(marker_pass)!r}).write_text('PASSED_AND_WROTE_HOST', encoding='utf-8')\n" + "def verify(x):\n" + " return x == 1\n" + ) + rec_pass = run_checker_oracle(malicious_pass_py, entry_fn="verify", valid_cases=(), allow_static_lint_only=True) + expect_false(rec_pass.passed, "pathlib import must be rejected") + expect_in("FORBIDDEN_CHECKER_IMPORT", {f["rule"] for f in rec_pass.findings}) + expect_false(marker_pass.exists(), "pathlib checker must never execute on host") + + marker_rej = tmp_path / "HOST_EXECUTED_DESPITE_REJECT.txt" + malicious_rej_py = ( + "import pathlib\n" + f"pathlib.Path({str(marker_rej)!r}).write_text('EXECUTED_DESPITE_STATIC_REJECT', encoding='utf-8')\n" + "def verify(x):\n" + " return float(x) == 1.0\n" + ) + rec_rej = run_checker_oracle(malicious_rej_py, entry_fn="verify", valid_cases=()) + expect_false(rec_rej.passed) + expect_false(marker_rej.exists(), "statically rejected checker must never execute on host") + + marker_hop = tmp_path / "HOST_EXECUTED_TYPING_SYS.txt" + typing_hop_py = ( + "import typing\n" + "_io = typing.sys.modules.get('io') or typing.sys.modules.get('_io')\n" + "if _io:\n" + f" with _io.open({str(marker_hop)!r}, 'w') as fh:\n" + " fh.write('PWN_TYPING_SYS')\n" + "def verify(x):\n" + " return x == 1\n" + ) + rec_hop = run_checker_oracle(typing_hop_py, entry_fn="verify", valid_cases=(), allow_static_lint_only=True) + expect_false(rec_hop.passed, "typing.sys.modules['_io'] module-hop must be rejected") + expect_in("FORBIDDEN_CHECKER_IO_ATTR", {f["rule"] for f in rec_hop.findings}) + expect_false(marker_hop.exists(), "typing.sys module-hop must never execute on host") + + +@requires_dep("soko") +@requires_lean() +def test_neon_c2_family_a_theorem_substitution_and_comment_mention_blocked() -> None: + val_lines = (repo_root() / "datasets" / "validation.jsonl").read_text(encoding="utf-8").splitlines() + fam_a_item = next(json.loads(ln) for ln in val_lines if json.loads(ln)["family"] == "A") + harness_a = json.loads((repo_root() / "datasets" / fam_a_item["harness_path"]).read_text(encoding="utf-8")) + thm_short = str(harness_a["theorem_name"]).split(".")[-1] + + # 1. Retyped trivial theorem with valid def sumOdd + bogus_retyped = ( + "```lean\nnamespace PubAV01\n" + "def sumOdd : Nat → Nat\n | 0 => 0\n | n + 1 => sumOdd n + (2 * n + 1)\n" + f"theorem {thm_short} : 1 + 1 = 2 := rfl\nend PubAV01\n```" + ) + inf_retyped = default_inference_oracle( + {"id": fam_a_item["id"], "family": "A", "prompt": fam_a_item["prompt"]}, + bogus_retyped, + ) + grad_retyped = default_dataset_item_grader(fam_a_item, bogus_retyped) + expect_false(inf_retyped["passed"], f"retyped theorem must fail inference oracle: {inf_retyped}") + expect_false(grad_retyped["passed"], f"retyped theorem must fail dataset grader: {grad_retyped}") + + # 2. Comment-only theorem mention + bogus_comment = f"```lean\n-- {thm_short}\ntheorem unrelated_trivial : 2 + 2 = 4 := rfl\n```" + inf_comment = default_inference_oracle( + {"id": fam_a_item["id"], "family": "A", "prompt": fam_a_item["prompt"]}, + bogus_comment, + ) + grad_comment = default_dataset_item_grader(fam_a_item, bogus_comment) + expect_false(inf_comment["passed"], "comment-only theorem mention must fail inference oracle") + expect_false(grad_comment["passed"], "comment-only theorem mention must fail dataset grader") + + # 3. Genuine positive completion with alpha-renamed theorem binder variable must still pass both! + alpha_valid = ( + "```lean\nnamespace PubAV01\n" + "def sumOdd : Nat → Nat\n | 0 => 0\n | n + 1 => sumOdd n + (2 * n + 1)\n" + "theorem sumOdd_eq_sq (m : Nat) : sumOdd m = m * m := by\n" + " induction m with\n | zero => rfl\n" + " | succ k ih =>\n simp only [sumOdd, ih]\n" + " rw [Nat.add_mul, Nat.mul_add, Nat.one_mul, Nat.mul_one]\n omega\n" + "end PubAV01\n```" + ) + inf_valid = default_inference_oracle( + {"id": fam_a_item["id"], "family": "A", "prompt": fam_a_item["prompt"]}, + alpha_valid, + ) + grad_valid = default_dataset_item_grader(fam_a_item, alpha_valid) + expect_true(inf_valid["passed"], f"alpha-renamed valid proof rejected by inference oracle: {inf_valid}") + expect_true(grad_valid["passed"], f"alpha-renamed valid proof rejected by dataset grader: {grad_valid}") + + +def test_neon_c3_lean_harness_trust0_pinned_defs_and_weak_statements() -> None: + cmd = build_capped_lean_args("/usr/bin/lean", Path("Test.lean")) + expect_in("--trust=0", cmd) + + prompt_pm = ( + "def vecDot : List Int → List Int → Int\n" + " | [], _ => 0\n | _, [] => 0\n | x :: xs, y :: ys => x * y + vecDot xs ys\n" + ) + mutated_pm = "def vecDot : List Int → List Int → Int\n | _, _ => 0\n" + expect_true("vecDot" in extract_normalized_lean_defs(prompt_pm)) + expect_in("TARGET_DEF_BODY_MUTATED", {f["rule"] for f in verify_pinned_lean_defs(prompt_pm, mutated_pm)}) + + prompt_eq = "def clampNat (lo hi x : Nat) : Nat := if x < lo then lo else if hi < x then hi else x\n" + cand_abbrev = "abbrev clampNat (lo hi x : Nat) : Nat := lo\n" + cand_ws = "def clampNat (lo hi x: Nat) : Nat := if x < lo then lo else if hi < x then hi else x\n" + expect_in("TARGET_DEF_BODY_MUTATED", {f["rule"] for f in verify_pinned_lean_defs(prompt_eq, cand_abbrev)}) + expect_eq(verify_pinned_lean_defs(prompt_eq, cand_ws), [], "binder whitespace must not false-positive") + + expect_in("VACUOUS_TRUE_CONCLUSION", {f["rule"] for f in audit_weak_statements("theorem t : (True) := trivial\n")}) + expect_in( + "VACUOUS_FALSE_HYPOTHESIS", + {f["rule"] for f in audit_weak_statements("theorem t : (False) → 1 = 2 := fun h => False.elim h\n")}, + ) + expect_in( + "VACUOUS_FALSE_HYPOTHESIS", + {f["rule"] for f in audit_weak_statements("theorem t (h : True → False) : 1 = 2 := False.elim (h trivial)\n")}, + ) + + +def test_neon_c4_nested_comments_period2_oscillation_and_chunk_rotation() -> None: + nested_1 = "theorem t : 1 = 1 := /- outer /- inner -/ still comment A -/ rfl" + nested_2 = "theorem t : 1 = 1 := /- outer /- inner -/ still comment B -/ rfl" + expect_eq(_strip_lean_comments(nested_1), "theorem t : 1 = 1 := rfl") + expect_eq( + compute_candidate_state_hash("A", f"```lean\n{nested_1}\n```"), + compute_candidate_state_hash("A", f"```lean\n{nested_2}\n```"), + ) + + osc_calls: list[int] = [] + + def _osc_inf(_pv: object, _comp: str) -> dict[str, object]: + idx = len(osc_calls) + osc_calls.append(idx) + rule = "RULE_X" if idx % 2 == 0 else "RULE_Y" + return {"passed": False, "epistemic_status": "REJECTED", "findings": [{"rule": rule, "line": 1, "message": rule}]} + + osc_res = run_repair_item( + {"id": "OSC-1", "family": "A", "prompt": "Prove something"}, + lambda _p, t: f"```lean\ntheorem t{t} : 1 = 1 := rfl\n```", + max_turns=4, + inference_oracle_fn=_osc_inf, + eval_grader_fn=lambda _it, _c: {"passed": False, "findings": []}, + include_plane1_retrieval=False, + ) + expect_true(osc_res["stagnation_events"] >= 1, "period-2 error oscillation must trigger stagnation escape") + + multi_findings = [{"rule": "LEAN_COMPILE_ERROR", "message": f"e{i}"} for i in range(6)] + ret1 = retrieve_plane1_excerpt( + family="A", + prompt_text="Prove Lean 4 theorem with induction and omega", + prior_findings=multi_findings, + exclude_chunk_ids=(), + prefer_worked_examples=False, + ) + ret2 = retrieve_plane1_excerpt( + family="A", + prompt_text="Prove Lean 4 theorem with induction and omega", + prior_findings=multi_findings, + exclude_chunk_ids=ret1["chunk_ids"], + prefer_worked_examples=True, + ) + expect_eq(set(ret1["chunk_ids"]) & set(ret2["chunk_ids"]), set(), "excluded chunks must rotate cleanly") + + +@requires_dep("soko") +@requires_mde() +def test_neon_c4_family_b_entry_fn_ignores_generator_helpers() -> None: + fam_b_prompt = ( + "Given generator:\n```python\ndef check_input_bounds(n):\n return n > 0\n" + "def generate(n):\n return n * 2\n```\n" + "Implement `verify(n, out)` returning True iff `out == 2 * n`." + ) + valid_checker_b = ( + "```python\ndef verify(n, out):\n" + " return type(n) is int and type(out) is int and n > 0 and out == 2 * n\n```" + ) + res = default_inference_oracle({"id": "B-MIS", "family": "B", "prompt": fam_b_prompt}, valid_checker_b) + expect_true(res["passed"], f"valid `verify` rejected due to generator `check_input_bounds`: {res}") + + +def test_neon_d1_d2_d5_domain_protocol_allowlist_and_fail_closed_certs() -> None: + for mod_code in ( + "import posix\n", + "import _io\n", + "import nt\n", + "import _posixsubprocess\n", + "import typing\nx = typing.sys\n", + "import collections\nx = collections._sys\n", + "from json import codecs\n", + "from fractions import *\n", + "fn = open\n", + ): + rej = run_restricted_sandbox_cell(mod_code) + expect_false(rej["ok"], f"{mod_code!r} must be blocked by sandbox allowlist") + expect_true(rej["ast_rejected"]) + + expect_true(len(audit_sandbox_cell_ast("x = 2 ** -1\n", strict_exact_math=True)) > 0) + + bare_proven = audit_transition_certificate({"epistemic_status": "PROVEN"}) + expect_false(bare_proven["passed"], "bare PROVEN cert without tcb_pins/witness_or_proof_sha256 must fail") + bare_rules = {f["rule"] for f in bare_proven["findings"]} + expect_in("CERT_TCB_MANIFEST_UNLOCKED", bare_rules) + expect_in("CERT_INVALID_WITNESS_SHA256", bare_rules) + + trunc_dijkstra = audit_transition_certificate( + { + "epistemic_status": "PROVEN", + "certificate_kind": "EXACT_DIJKSTRA_OPTIMUM", + "truncated_by_budget": True, + "witness_or_proof_sha256": "a" * 64, + "tcb_pins": {"tcb_manifest_verified": True, "checker_sha256": "b" * 64}, + } + ) + expect_false(trunc_dijkstra["passed"], "truncated_by_budget=True cannot claim PROVEN EXACT_DIJKSTRA_OPTIMUM") + expect_in("CERT_TRUNCATED_SEARCH_BUDGET", {f["rule"] for f in trunc_dijkstra["findings"]}) diff --git a/verification/VERIFICATION_RECEIPT.json b/verification/VERIFICATION_RECEIPT.json index fc289b2..798c081 100644 --- a/verification/VERIFICATION_RECEIPT.json +++ b/verification/VERIFICATION_RECEIPT.json @@ -590,7 +590,7 @@ "lineage_and_pins": { "base_gguf_q8_0_sha256": "10cc391b403021dd11c614679d2fd92f611c3681d29e29651b717316965d61e1", "compiler_manifest_sha256": "feb435b67d3f065539cef8a5c02ab9f189a20314ca1388b845f268538f89935b", - "corpus_tree_sha256": "5bdb7f30363b956e82947f668f98c48f99876e87fb5ed4f785b873e4f4f98305", + "corpus_tree_sha256": "93624aae09eea99c3d35ebfa2e9acde1f9ac9c675167461652486850f206b642", "datasets_sha256sums_manifest_sha256": "312953bc99bb42e6cbef1d1a3c240975f2baf3c60506df6c9eef06a678d45d56", "lean_toolchain": "leanprover/lean4:v4.35.0", "llama_server_version": "0.5.0 (build 11146, commit 7fe450e19)", @@ -602,8 +602,8 @@ "plane1_formal_spec_sha256": "49beff2a0185cfa4032170b614a543c0d67541494e858803d21cfb837111b230", "public_holdout_commitment_sha256": "e7c9b9da52be68c4731e35961c31ec33decf440e4ccb00b2c9072e029054f032", "safetensors_checkpoint_fingerprint": "1ed4483f1d3ffbc3ffc7a26984be212e7cce455ac1e0651d1bcba1313ea2bd64", - "spec_lineage_json_sha256": "8307454318ff2cc75b0bf6a8448ceccf7923ee2b55526aced6a9f1d0c6cd7972", - "tracked_evidence_merkle_sha256": "4b4fe85230c5986844eb3a01bff4087710129bfb4d07d3ba821cf0ca9d2fdf39", + "spec_lineage_json_sha256": "344f5506e702dc8dc0ee7374d30af788e3d4c2c370fcdbc877435e133da26a76", + "tracked_evidence_merkle_sha256": "fea60d43fe59ba05f226d7f2b543c630fd6b9a8e4efc61817b9133aab5456e98", "vault_sha256sums_manifest_sha256": "b14cf1efeedf7a7c8c15ed57e587c3d18087f557e6ff1807068c4e9c9ef6f6ee", "zone_manifest_id": "sha256:4e00656cb47fce97f687ae638afba22c569b9680b3780eeb2e5ea71a62c55eca" }, diff --git a/verification/independent_rederive_p2b.py b/verification/independent_rederive_p2b.py index 96192a0..bf92523 100644 --- a/verification/independent_rederive_p2b.py +++ b/verification/independent_rederive_p2b.py @@ -462,8 +462,10 @@ def main() -> int: st_dir = _resolve_safetensors_dir() npz_path = _resolve_activations_npz() - # Load independent Plane 1 lineage and zone manifest - lineage_doc = json.loads((REPO_ROOT / "spec" / "lineage.json").read_text(encoding="utf-8")) + # Load independent Plane 1 lineage and zone manifest (using frozen v0.1 snapshot when verifying v0.1 P2-B) + v01_snap = EXPECTED_P2B_DIR / "v01_plane1_snapshot" + lineage_src = (v01_snap / "spec" / "lineage.json") if (v01_snap / "spec" / "lineage.json").is_file() else (REPO_ROOT / "spec" / "lineage.json") + lineage_doc = json.loads(lineage_src.read_text(encoding="utf-8")) zone_doc = json.loads((REPO_ROOT / "schemas" / "zone-manifest.v0.1.json").read_text(encoding="utf-8")) zone_manifest_id = str(zone_doc["manifest_id"]) if not hmac.compare_digest(zone_manifest_id, str(lineage_doc["zone_manifest_id"])): @@ -586,7 +588,10 @@ def main() -> int: # 6. Independent 5-layer BindingManifest ident = identify_checkpoint_independent(st_dir) - p1_hashes = {rel: sha256_file(REPO_ROOT / rel) for rel in sorted(PLANE1_RELPATHS)} + p1_hashes = { + rel: sha256_file((v01_snap / rel) if (v01_snap / rel).is_file() else (REPO_ROOT / rel)) + for rel in sorted(PLANE1_RELPATHS) + } recipe = { "compiler": "weft/0.1.0", "harvested_activations_meta_sha256": meta_sha256, diff --git a/verification/verify_compiler_artifacts.py b/verification/verify_compiler_artifacts.py index 9ed6cfb..d396aaf 100644 --- a/verification/verify_compiler_artifacts.py +++ b/verification/verify_compiler_artifacts.py @@ -178,14 +178,46 @@ def main() -> int: verified_hashes["p2b.certificate_file"] = sha256_file(cert_file) verified_hashes["p2b.binding_canonical"] = bind_obj.sha256() verified_hashes["p2b.binding_file"] = sha256_file(bind_file) - bind_res = verify_binding( - bind_obj, - checkpoint_dir=st_dir, - runtime_flags=weft_flags, - plane1_root=PLANE1_ROOT, - expected_factor_digest=p2b_cert.factor_digest, - expected_certificate_sha256=p2b_cert.sha256(), - ) + v01_snap = ART_DIR / "p2b" / "v01_plane1_snapshot" + if v01_snap.is_dir(): + import shutil as _shutil + import tempfile as _tempfile + with _tempfile.TemporaryDirectory(prefix="mcproof_v01_p1_") as _td: + _v01_root = Path(_td) + for _sub in ("spec", "schemas", "lean"): + _shutil.copytree(PLANE1_ROOT / _sub, _v01_root / _sub, ignore=_shutil.ignore_patterns(".lake")) + for _ov in v01_snap.rglob("*"): + if _ov.is_file(): + _rel = _ov.relative_to(v01_snap) + (_v01_root / _rel).parent.mkdir(parents=True, exist_ok=True) + _shutil.copy2(_ov, _v01_root / _rel) + bind_res = verify_binding( + bind_obj, + checkpoint_dir=st_dir, + runtime_flags=weft_flags, + plane1_root=_v01_root, + expected_factor_digest=p2b_cert.factor_digest, + expected_certificate_sha256=p2b_cert.sha256(), + ) + live_v02_bind = verify_binding( + bind_obj, + checkpoint_dir=st_dir, + runtime_flags=weft_flags, + plane1_root=PLANE1_ROOT, + expected_factor_digest=p2b_cert.factor_digest, + expected_certificate_sha256=p2b_cert.sha256(), + ) + if live_v02_bind.ok or "stale_lineage" not in live_v02_bind.problems: + defects.append("expected Hard Veto 2 stale_lineage when checking v0.1 P2-B binding against v0.2 Plane 1") + else: + bind_res = verify_binding( + bind_obj, + checkpoint_dir=st_dir, + runtime_flags=weft_flags, + plane1_root=PLANE1_ROOT, + expected_factor_digest=p2b_cert.factor_digest, + expected_certificate_sha256=p2b_cert.sha256(), + ) if not bind_res.ok: defects.append(f"verify_binding_failed:{bind_res.problems}") From e294eb3aaa1ea5904799e2f333ab91a42987496f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 20 Nov 2026 00:25:03 +0000 Subject: [PATCH 2/9] fix: document Lean axiom footprint and probe functional bwrap Expand README trust bullet with per-theorem standard-axiom breakdown from collectAxioms/#print axioms. Replace shutil.which-only bwrap detection with a minimal ro-bind probe plus a /run/user session check required by lake comparator; skip the sandboxed reference comparator test when that probe fails instead of failing on broken bubblewrap installs. Co-authored-by: ahraz --- README.md | 2 +- src/mcproof/lean_harness.py | 45 +++++++++++++++++++++++++++++++++- tests/test_lean_harness.py | 21 ++++++++++------ verification/verify_oracles.py | 4 +-- 4 files changed, 60 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index b8919c7..a9efe3a 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ Run the full Tier 2 oracle gates and Family `A` to `D` graders. On Linux, `lake ## What you must trust -- Lean `v4.35.0` and its five bundled external kernels (`leanchecker-paranoid`, `lean4lean`, `nanoda`, `con-leche`, `con-ron`), plus the Lean compiler and C runtime for native binaries (`Quot.sound` and `propext` measured via `collectAxioms` across all `29` reference theorems, with zero `Classical.choice`). +- Lean `v4.35.0` and its five bundled external kernels (`leanchecker-paranoid`, `lean4lean`, `nanoda`, `con-leche`, `con-ron`), plus the Lean compiler and C runtime for native binaries. Across the `29` reference theorems, `collectAxioms` and `#print axioms` on `Mcproof.Proofs` use only `{propext, Classical.choice, Quot.sound}`. Axiom-free: `bezoutCheck_sound`, `bezoutCheck_complete`, `ratLeCross_sound`, `scalar_projection_retention_illusion`. Also `Classical.choice`: `clampNat_in_range`, `cleanSegments_safe`, `cleanSegments_singleton`, `prefundedVal_bounded_drop_witness`. The other `21` use `propext` and/or `Quot.sound` only (no custom axioms). - Python 3 exact `int` and `fractions.Fraction` arithmetic (`-I -O -B -W error`), `numpy` CPU `float64` linear algebra, and the Rust `tree-sitter` CST parser in `siliconcordat/mde`. - Host OS sandboxing (`bwrap` network-unshared read-only root on Linux, or subprocess `-I` environment isolation without OS syscall confinement on macOS and Windows) and SHA-256 digest implementations. diff --git a/src/mcproof/lean_harness.py b/src/mcproof/lean_harness.py index bac8b12..0c5e74a 100644 --- a/src/mcproof/lean_harness.py +++ b/src/mcproof/lean_harness.py @@ -78,6 +78,49 @@ r"(?Ptheorem|lemma)[ \t]+(?P[A-Za-z_][A-Za-z0-9_.']*)\b(?P
.*?)(?::=|\bwhere\b|\|)" ) + +def linux_bwrap_functional(*, timeout_sec: float = 5.0) -> bool: + """Return True when Linux `bwrap` runs a minimal sandbox and can create `lake comparator`'s `/run/user` session dir.""" + if not sys.platform.startswith("linux"): + return False + bwrap_bin = shutil.which("bwrap") + if not bwrap_bin: + return False + try: + basic = subprocess.run( + [bwrap_bin, "--ro-bind", "/", "/", "true"], + capture_output=True, + timeout=timeout_sec, + check=False, + ) + if basic.returncode != 0: + return False + uid = os.getuid() + lake_compat = subprocess.run( + [ + bwrap_bin, + "--ro-bind", + "/", + "/", + "--bind", + "/run", + "/run", + "--bind", + "/tmp", + "/tmp", + "sh", + "-c", + f"mkdir -p /run/user/{uid} && test -w /run/user/{uid}", + ], + capture_output=True, + timeout=timeout_sec, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return False + return lake_compat.returncode == 0 + + AXIOM_AUDIT_TEMPLATE = """import Lean import {module_name} @@ -1317,7 +1360,7 @@ def verify_lean4_project_comparator( }) # Fail closed before compiling untrusted Lean code when Linux bubblewrap is unavailable unless explicitly opted in - bwrap_available = shutil.which("bwrap") is not None and sys.platform.startswith("linux") + bwrap_available = linux_bwrap_functional() caveats: list[str] = [] if not bwrap_available: if not allow_unsandboxed: diff --git a/tests/test_lean_harness.py b/tests/test_lean_harness.py index b823627..64d0787 100644 --- a/tests/test_lean_harness.py +++ b/tests/test_lean_harness.py @@ -14,6 +14,7 @@ audit_weak_statements, classify_host_resource_failure, lint_lean4_source, + linux_bwrap_functional, parse_comparator_stdout, scrub_challenge_artifacts, verify_lean4_project_comparator, @@ -114,8 +115,7 @@ def test_lean_env_uses_os_pathsep(monkeypatch: pytest.MonkeyPatch, tmp_path: Pat def test_project_comparator_fails_closed_when_bwrap_unavailable_without_opt_in( monkeypatch: pytest.MonkeyPatch, ) -> None: - orig_which = shutil.which - monkeypatch.setattr(shutil, "which", lambda name, *a, **kw: None if name == "bwrap" else orig_which(name, *a, **kw)) + monkeypatch.setattr("mcproof.lean_harness.linux_bwrap_functional", lambda **kw: False) res = verify_lean4_project_comparator(repo_root() / "lean", allow_unsandboxed=False) expect_false(res.passed, "comparator must fail closed without bwrap when allow_unsandboxed=False") expect_in("SANDBOX_UNAVAILABLE", {f["rule"] for f in res.findings}) @@ -124,16 +124,21 @@ def test_project_comparator_fails_closed_when_bwrap_unavailable_without_opt_in( @requires_dep("soko") @requires_lean() def test_reference_lean_project_passes_six_kernel_comparator() -> None: - has_bwrap = shutil.which("bwrap") is not None and sys.platform.startswith("linux") - res = verify_lean4_project_comparator(repo_root() / "lean", allow_unsandboxed=not has_bwrap) + if not sys.platform.startswith("linux"): + pytest.skip("sandboxed `lake comparator --paranoid` requires Linux bubblewrap") + if not linux_bwrap_functional(): + pytest.skip( + "Linux bwrap is missing or non-functional on this host " + "(minimal `bwrap --ro-bind / / true` or lake `/run/user` session probe failed)" + ) + res = verify_lean4_project_comparator(repo_root() / "lean", allow_unsandboxed=False) expect_true(res.passed, f"reference lean project failed: {res.findings}") expect_eq(len(res.paranoid_kernels_run), 6, "all 6 bundled Lean 4.35.0 kernels must accept") expect_eq(len(res.theorems_checked), 29, "all 29 theorems in comparator.json must be checked") expect_eq(len(res.sorry_declarations), 0, "zero sorry declarations allowed in Mcproof.Proofs") expect_eq(len(res.nonstandard_axioms), 0, "zero non-standard axioms allowed") expect_eq(res.challenge_sorry_whitelisted_module, "Mcproof.Spec") - if not has_bwrap: - expect_in("UNSANDBOXED_EXECUTION", res.caveats) + expect_true(res.sandboxed, "reference project comparator must run inside a working bwrap sandbox") @requires_dep("soko") @@ -218,7 +223,7 @@ def checkedGate (s : String) : Bool := honestGate s @requires_dep("soko") @requires_lean() def test_rejects_planted_cheat_in_project_comparator(tmp_path: Path) -> None: - has_bwrap = shutil.which("bwrap") is not None and sys.platform.startswith("linux") + allow_unsandboxed = not linux_bwrap_functional() work = tmp_path / "lean_copy" shutil.copytree( repo_root() / "lean", @@ -230,7 +235,7 @@ def test_rejects_planted_cheat_in_project_comparator(tmp_path: Path) -> None: proofs_file.read_text(encoding="utf-8") + "\naxiom planted_cheat : False\n", encoding="utf-8", ) - res = verify_lean4_project_comparator(work, allow_unsandboxed=not has_bwrap) + res = verify_lean4_project_comparator(work, allow_unsandboxed=allow_unsandboxed) expect_false(res.passed, "project comparator must reject planted axiom") rules = {f["rule"] for f in res.findings} expect_in("EXTRA_AXIOM_DECL", rules) diff --git a/verification/verify_oracles.py b/verification/verify_oracles.py index 0850b8d..d45f5a4 100644 --- a/verification/verify_oracles.py +++ b/verification/verify_oracles.py @@ -21,7 +21,7 @@ OUT_JSON = _VERIF_DIR / "oracles_verification_receipt.json" sys.path.insert(0, str(REPO_ROOT / "src")) -from mcproof.lean_harness import verify_lean4_project_comparator, verify_lean4_source # noqa: E402 +from mcproof.lean_harness import linux_bwrap_functional, verify_lean4_project_comparator, verify_lean4_source # noqa: E402 from mcproof.lineage import verify_lineage_manifest # noqa: E402 from mcproof.mutation import run_f0_gate_and_corruptions # noqa: E402 from mcproof.oracles import run_checker_oracle # noqa: E402 @@ -121,7 +121,7 @@ def main() -> int: report["defects"].append(f"pytest failed under -O -B -W error: {pytest_proc.stdout[-400:]} {pytest_proc.stderr[-400:]}") # 4. Independent Lean 4 `lake build` + 6-kernel `lake comparator --paranoid` + sandbox gate + adversarial cheats - has_bwrap = shutil.which("bwrap") is not None and sys.platform.startswith("linux") + has_bwrap = linux_bwrap_functional() if not has_bwrap: fail_closed_sb = verify_lean4_project_comparator(REPO_ROOT / "lean", allow_unsandboxed=False) sb_blocked = (not fail_closed_sb.passed) and any(f["rule"] == "SANDBOX_UNAVAILABLE" for f in fail_closed_sb.findings) From 71d1a58afc7fb0f77749a3f56ac2609bce3ba814 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 20 Nov 2026 00:40:04 +0000 Subject: [PATCH 3/9] chore: regenerate VERIFICATION_RECEIPT after v0.1 merge Rebuild final receipt so P2-B public validation metrics match the honest scorecard (-0.045455 vs B0) after merging feat/formal-verification-mde-v0.1. Co-authored-by: ahraz --- verification/VERIFICATION_RECEIPT.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verification/VERIFICATION_RECEIPT.json b/verification/VERIFICATION_RECEIPT.json index 2ea2d1c..aab6232 100644 --- a/verification/VERIFICATION_RECEIPT.json +++ b/verification/VERIFICATION_RECEIPT.json @@ -650,7 +650,7 @@ "control_ordering_summary": "P2-A 0/48 [0.00%] < P2-B 2/48 [4.17%] < B0 4/48 [8.33%] = goodhart 4/48 [8.33%] < constant logit_bias 6/48 [12.50%] < B1 8/48 [16.67%]", "efficacy_verdict": "REJECTED_NEGATIVE_EFFICACY", "p2b_passed": 2, - "p2b_relative_failure_reduction_vs_b0": -1.0, + "p2b_relative_failure_reduction_vs_b0": -0.045455, "passed": false, "total": 48 }, From 3e634b93b3b2b62f3b13bb9acdfab2507ab03362 Mon Sep 17 00:00:00 2001 From: ahraz <38987266+ahrazzle@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:20:15 -0600 Subject: [PATCH 4/9] Use fullmatch for anchored digest, commit and rational patterns The patterns end in $, which also matches before a final newline, so a value with one trailing newline was accepted. Parsed values were still correct. Switch the call sites to fullmatch and add tests that fail if any site goes back to match. --- src/mcproof/domain_protocol.py | 6 ++-- src/mcproof/lineage.py | 4 +-- tests/test_regex_fullmatch.py | 61 ++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 tests/test_regex_fullmatch.py diff --git a/src/mcproof/domain_protocol.py b/src/mcproof/domain_protocol.py index e18ff2e..6cbe0ee 100644 --- a/src/mcproof/domain_protocol.py +++ b/src/mcproof/domain_protocol.py @@ -732,7 +732,7 @@ def audit_transition_certificate( ) else: chk_sha = str(pins.get("checker_sha256") or pins.get("checker_auditor_py_sha256") or "") - if not pins.get("tcb_manifest_verified", False) or not SHA256_64_RE.match(chk_sha): + if not pins.get("tcb_manifest_verified", False) or not SHA256_64_RE.fullmatch(chk_sha): findings.append( { "rule": "CERT_TCB_MANIFEST_UNLOCKED", @@ -845,7 +845,7 @@ def audit_transition_certificate( # F-08: Missing, malformed non-64-hex, or unregistered witness_or_proof_sha256 (fail closed, NEON-D5) if status in ("VERIFIED", "PROVEN"): w_sha = str(cert.get("witness_or_proof_sha256", "")) - if not SHA256_64_RE.match(w_sha): + if not SHA256_64_RE.fullmatch(w_sha): findings.append( { "rule": "CERT_INVALID_WITNESS_SHA256", @@ -909,7 +909,7 @@ def _parse_exact_rational(val: Any, *, max_abs_int: int = MAX_CERT_ABS_INT) -> F if abs(val) > max_abs_int: raise ValueError("integer exceeds cap") return Fraction(val, 1) - if isinstance(val, str) and STRICT_RATIONAL_RE.match(val): + if isinstance(val, str) and STRICT_RATIONAL_RE.fullmatch(val): if "/" in val: n_str, d_str = val.split("/", 1) num, den = int(n_str), int(d_str) diff --git a/src/mcproof/lineage.py b/src/mcproof/lineage.py index 5fe72d8..4b2155a 100644 --- a/src/mcproof/lineage.py +++ b/src/mcproof/lineage.py @@ -202,7 +202,7 @@ def build_lineage_manifest( upstream_clean: dict[str, dict[str, Any]] = {} for repo_name, info in sorted(upstream_raw.items()): commit = str(info["commit"]) - if not COMMIT_RE.match(commit): + if not COMMIT_RE.fullmatch(commit): raise StaleLineageError(f"upstream repo {repo_name} commit is not 40-hex: {commit!r}") upstream_clean[repo_name] = { "url": str(info["url"]), @@ -292,7 +292,7 @@ def _git_head_sha(repo_path: Path) -> str | None: if proc.returncode != 0: return None head = proc.stdout.strip() - return head if COMMIT_RE.match(head) else None + return head if COMMIT_RE.fullmatch(head) else None def verify_lineage_manifest( diff --git a/tests/test_regex_fullmatch.py b/tests/test_regex_fullmatch.py new file mode 100644 index 0000000..063eb10 --- /dev/null +++ b/tests/test_regex_fullmatch.py @@ -0,0 +1,61 @@ +"""Anchored digest, commit and rational patterns must reject a trailing newline.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from mcproof.domain_protocol import _parse_exact_rational, audit_transition_certificate +from mcproof.lineage import StaleLineageError, build_lineage_manifest +from mcproof.paths import repo_root + +from helpers import expect_eq, expect_in + +HEX64 = "a" * 64 + + +def _rules(cert: dict) -> list[str]: + return [f["rule"] for f in audit_transition_certificate(cert)["findings"]] + + +def test_canonical_rational_still_parses() -> None: + expect_eq(str(_parse_exact_rational("3/4")), "3/4") + expect_eq(str(_parse_exact_rational("-5")), "-5") + + +def test_rational_trailing_newline_rejected() -> None: + with pytest.raises(ValueError): + _parse_exact_rational("3/4\n") + + +def test_witness_digest_trailing_newline_rejected() -> None: + ok = {"epistemic_status": "VERIFIED", "witness_or_proof_sha256": HEX64, + "tcb_pins": {"tcb_manifest_verified": True, "checker_sha256": HEX64}} + expect_eq(_rules(ok), []) + bad_w = {**ok, "witness_or_proof_sha256": HEX64 + "\n"} + expect_in("CERT_INVALID_WITNESS_SHA256", _rules(bad_w)) + + +def test_tcb_checker_digest_trailing_newline_rejected() -> None: + bad = {"epistemic_status": "VERIFIED", "witness_or_proof_sha256": HEX64, + "tcb_pins": {"tcb_manifest_verified": True, "checker_sha256": HEX64 + "\n"}} + expect_in("CERT_TCB_MANIFEST_UNLOCKED", _rules(bad)) + + +def test_upstream_commit_trailing_newline_rejected(tmp_path: Path) -> None: + root = repo_root() + lin = json.loads((root / "spec" / "lineage.json").read_text(encoding="utf-8")) + repos = {} + for k, v in lin["upstream_repos"].items(): + repos[k] = {"dir": k.split("/")[-1], "url": v["url"], "commit": v["commit"], "roles": v["roles"]} + deps = tmp_path / "deps" + deps.mkdir() + (deps / "DEPS_MANIFEST.json").write_text(json.dumps({"repos": repos}), encoding="utf-8") + build_lineage_manifest(root, d_dir=deps) + first = sorted(repos)[0] + repos[first]["commit"] += "\n" + (deps / "DEPS_MANIFEST.json").write_text(json.dumps({"repos": repos}), encoding="utf-8") + with pytest.raises(StaleLineageError): + build_lineage_manifest(root, d_dir=deps) From 7b60918d561f1c338adcd253be5b10d286b95971 Mon Sep 17 00:00:00 2001 From: ahraz <38987266+ahrazzle@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:39:41 -0600 Subject: [PATCH 5/9] audit: reject a certificate that names no recognized status An empty object and an unknown status produced no findings, and the auditor reported VERIFIED. A rejected or conjectural record was relabeled the same way. Those records now keep their own status, and a missing or unknown status is a finding. --- src/mcproof/domain_protocol.py | 28 +++++++++- tests/test_audit_status_fail_closed.py | 76 ++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 tests/test_audit_status_fail_closed.py diff --git a/src/mcproof/domain_protocol.py b/src/mcproof/domain_protocol.py index e18ff2e..36b8e84 100644 --- a/src/mcproof/domain_protocol.py +++ b/src/mcproof/domain_protocol.py @@ -51,6 +51,9 @@ "UNVERIFIED_CALLER_BOUNDS", } ) +RECOGNIZED_CERT_STATUSES: frozenset[str] = NON_VERIFIED_SUBTIERS | frozenset( + {"VERIFIED", "PROVEN", "REJECTED"} +) ALLOWLISTED_SANDBOX_MODULES: frozenset[str] = frozenset( { @@ -643,6 +646,22 @@ def audit_transition_certificate( findings: list[dict[str, str]] = [] status = str(cert.get("epistemic_status", "")).strip().upper() proof_mode = str(cert.get("proof_mode", "")).strip().lower() + + # An omitted or invented status has no checks to fail, so it must not pass. + if not status: + findings.append( + { + "rule": "CERT_STATUS_ABSENT", + "message": "Certificate omits epistemic_status.", + } + ) + elif status not in RECOGNIZED_CERT_STATUSES: + findings.append( + { + "rule": "CERT_STATUS_UNKNOWN", + "message": f"epistemic_status {status!r} is not a recognized certificate status.", + } + ) prior_status = str(cert.get("prior_status", "")).strip().upper() evidence_kind = str(cert.get("evidence_kind", "")).strip().lower() @@ -861,9 +880,16 @@ def audit_transition_certificate( ) passed = len(findings) == 0 + # Do not relabel a lower bound, a conjecture, or a rejection as VERIFIED. + if not passed: + out_status = "REJECTED" + elif status in ("VERIFIED", "PROVEN"): + out_status = "VERIFIED" + else: + out_status = status return { "passed": passed, - "epistemic_status": "VERIFIED" if passed else "REJECTED", + "epistemic_status": out_status, "finding_count": len(findings), "findings": findings, } diff --git a/tests/test_audit_status_fail_closed.py b/tests/test_audit_status_fail_closed.py new file mode 100644 index 0000000..738417b --- /dev/null +++ b/tests/test_audit_status_fail_closed.py @@ -0,0 +1,76 @@ +"""An empty or unknown certificate must not come back VERIFIED. + +The auditor used to return VERIFIED whenever it found no red-flag field. +A certificate with no status has no such field, so `{}` passed. The same +path relabeled REJECTED and CONJECTURE as VERIFIED. +""" + +from __future__ import annotations + +import io +import json +import sys +from pathlib import Path + +from mcproof.cli import main as cli_main +from mcproof.domain_protocol import audit_transition_certificate + +from helpers import expect_eq, expect_false, expect_in, expect_true + +HEX64 = "a" * 64 + + +def _rules(cert: dict) -> list[str]: + return [f["rule"] for f in audit_transition_certificate(cert)["findings"]] + + +def test_empty_certificate_is_rejected() -> None: + result = audit_transition_certificate({}) + expect_false(result["passed"]) + expect_eq(result["epistemic_status"], "REJECTED") + expect_in("CERT_STATUS_ABSENT", _rules({})) + + +def test_unknown_status_is_rejected() -> None: + cert = {"epistemic_status": "BANANA"} + result = audit_transition_certificate(cert) + expect_false(result["passed"]) + expect_eq(result["epistemic_status"], "REJECTED") + expect_in("CERT_STATUS_UNKNOWN", _rules(cert)) + + +def test_rejected_and_conjecture_are_not_relabeled_verified() -> None: + rejected = audit_transition_certificate({"epistemic_status": "REJECTED"}) + expect_true(rejected["passed"], f"a plain rejection is not an illegal claim: {rejected}") + expect_eq(rejected["epistemic_status"], "REJECTED") + + conjecture = audit_transition_certificate({"epistemic_status": "CONJECTURE"}) + expect_true(conjecture["passed"], f"CONJECTURE does not require pins: {conjecture}") + expect_eq(conjecture["epistemic_status"], "CONJECTURE") + + +def test_verified_with_pins_still_passes() -> None: + cert = { + "epistemic_status": "VERIFIED", + "witness_or_proof_sha256": HEX64, + "tcb_pins": {"tcb_manifest_verified": True, "checker_sha256": HEX64}, + } + result = audit_transition_certificate(cert) + expect_true(result["passed"], f"pinned VERIFIED certificate rejected: {result}") + expect_eq(result["epistemic_status"], "VERIFIED") + + +def test_cli_rejects_empty_certificate(tmp_path: Path) -> None: + path = tmp_path / "empty.json" + path.write_text("{}\n", encoding="utf-8") + old = sys.stdout + sys.stdout = io.StringIO() + try: + code = cli_main(["--json", "audit-cert", str(path)]) + raw = sys.stdout.getvalue() + finally: + sys.stdout = old + expect_eq(code, 1) + parsed = json.loads(raw) + expect_eq(parsed["epistemic_status"], "REJECTED") + expect_false(parsed["passed"]) From a37a90282428753f5e4b36978aef7025f1d912dc Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 20 Nov 2026 09:32:13 +0000 Subject: [PATCH 6/9] Restore KNOWN, REFUSED, and IN_PROGRESS as recognized certificate statuses. PR #11 added fail-closed status validation but omitted these three tiers from RECOGNIZED_CERT_STATUSES, so clean certificates were rejected with CERT_STATUS_UNKNOWN. Keep absent and invented statuses rejected, and add regression tests that each tier passes audit without relabeling to VERIFIED. Co-authored-by: ahraz --- src/mcproof/domain_protocol.py | 2 +- tests/test_audit_status_fail_closed.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/mcproof/domain_protocol.py b/src/mcproof/domain_protocol.py index 494cc6d..9165400 100644 --- a/src/mcproof/domain_protocol.py +++ b/src/mcproof/domain_protocol.py @@ -52,7 +52,7 @@ } ) RECOGNIZED_CERT_STATUSES: frozenset[str] = NON_VERIFIED_SUBTIERS | frozenset( - {"VERIFIED", "PROVEN", "REJECTED"} + {"VERIFIED", "PROVEN", "REJECTED", "KNOWN", "REFUSED", "IN_PROGRESS"} ) ALLOWLISTED_SANDBOX_MODULES: frozenset[str] = frozenset( diff --git a/tests/test_audit_status_fail_closed.py b/tests/test_audit_status_fail_closed.py index 738417b..8f17f45 100644 --- a/tests/test_audit_status_fail_closed.py +++ b/tests/test_audit_status_fail_closed.py @@ -39,6 +39,16 @@ def test_unknown_status_is_rejected() -> None: expect_in("CERT_STATUS_UNKNOWN", _rules(cert)) +def test_known_refused_and_in_progress_are_recognized() -> None: + for tier in ("KNOWN", "REFUSED", "IN_PROGRESS"): + cert = {"epistemic_status": tier} + result = audit_transition_certificate(cert) + expect_true(result["passed"], f"{tier} certificate rejected: {result}") + expect_eq(result["epistemic_status"], tier) + rules = _rules(cert) + expect_false("CERT_STATUS_UNKNOWN" in rules, f"unexpected CERT_STATUS_UNKNOWN for {tier}: {rules}") + + def test_rejected_and_conjecture_are_not_relabeled_verified() -> None: rejected = audit_transition_certificate({"epistemic_status": "REJECTED"}) expect_true(rejected["passed"], f"a plain rejection is not an illegal claim: {rejected}") From 3360fa5b227744a3bbffb883828210b9b19f1e4a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 20 Nov 2026 10:24:04 +0000 Subject: [PATCH 7/9] Reject a lower tier promoted to KNOWN under the A7 tier swap. audit_transition_certificate already blocks LINT_PASS, LOWER_BOUND, and CONJECTURE from becoming VERIFIED or PROVEN. KNOWN now uses that same check and the same CERT_A7_ILLEGAL_TIER_SWAP finding. Co-authored-by: ahraz --- src/mcproof/domain_protocol.py | 5 +++-- tests/test_audit_status_fail_closed.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/mcproof/domain_protocol.py b/src/mcproof/domain_protocol.py index 9165400..79094c2 100644 --- a/src/mcproof/domain_protocol.py +++ b/src/mcproof/domain_protocol.py @@ -665,8 +665,9 @@ def audit_transition_certificate( prior_status = str(cert.get("prior_status", "")).strip().upper() evidence_kind = str(cert.get("evidence_kind", "")).strip().lower() - # A7: Reject tier-swap promotions of LINT_PASS / LOWER_BOUND / bounded-subgraph to VERIFIED or PROVEN - if status in ("VERIFIED", "PROVEN"): + # A7: Reject tier-swap promotions of LINT_PASS / LOWER_BOUND / bounded-subgraph + # to VERIFIED, PROVEN, or KNOWN. KNOWN uses this same evidence check. + if status in ("VERIFIED", "PROVEN", "KNOWN"): if prior_status in NON_VERIFIED_SUBTIERS or evidence_kind in ("static_lint", "lower_bound_only", "bounded_subgraph"): findings.append( { diff --git a/tests/test_audit_status_fail_closed.py b/tests/test_audit_status_fail_closed.py index 8f17f45..9228a3a 100644 --- a/tests/test_audit_status_fail_closed.py +++ b/tests/test_audit_status_fail_closed.py @@ -59,6 +59,31 @@ def test_rejected_and_conjecture_are_not_relabeled_verified() -> None: expect_eq(conjecture["epistemic_status"], "CONJECTURE") +def test_conjecture_to_known_without_evidence_is_rejected() -> None: + """A7 rejects a lower tier promoted to KNOWN with no replayed witness. + + Reverting the KNOWN arm of the tier check lets this certificate pass. + """ + cert = {"epistemic_status": "KNOWN", "prior_status": "CONJECTURE"} + result = audit_transition_certificate(cert) + expect_false(result["passed"], f"CONJECTURE to KNOWN was accepted: {result}") + expect_eq(result["epistemic_status"], "REJECTED") + expect_in("CERT_A7_ILLEGAL_TIER_SWAP", _rules(cert)) + + +def test_known_with_accepted_evidence_keeps_known() -> None: + """KNOWN carrying the witness pins A7 already accepts for VERIFIED stays KNOWN.""" + cert = { + "epistemic_status": "KNOWN", + "witness_or_proof_sha256": HEX64, + "tcb_pins": {"tcb_manifest_verified": True, "checker_sha256": HEX64}, + } + result = audit_transition_certificate(cert) + expect_true(result["passed"], f"KNOWN with witness evidence rejected: {result}") + expect_eq(result["epistemic_status"], "KNOWN") + expect_false("CERT_A7_ILLEGAL_TIER_SWAP" in _rules(cert), result) + + def test_verified_with_pins_still_passes() -> None: cert = { "epistemic_status": "VERIFIED", From ba244c2c9920513d2431d8d91abe7b610632af2d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 20 Nov 2026 10:53:52 +0000 Subject: [PATCH 8/9] Require a source, reference, and artifact digest for KNOWN. A KNOWN certificate now carries citation.source, citation.reference, and citation.artifact_sha256. A proof digest and tcb_pins do not satisfy it. A lower prior is still an A7 tier swap when a citation is present. Co-authored-by: ahraz --- src/mcproof/domain_protocol.py | 87 +++++++++++++++++++- tests/test_audit_status_fail_closed.py | 106 ++++++++++++++++++++++--- 2 files changed, 181 insertions(+), 12 deletions(-) diff --git a/src/mcproof/domain_protocol.py b/src/mcproof/domain_protocol.py index 79094c2..a6111b0 100644 --- a/src/mcproof/domain_protocol.py +++ b/src/mcproof/domain_protocol.py @@ -9,7 +9,8 @@ sandbox usable by any `DomainOracle` closed-loop repair runner. 4. `audit_transition_certificate`, `verify_dual_product_parity`, and `audit_portfolio_bounds`: deterministic soundness checkers enforcing non-tautological `must_fail_controls` (`A6`), rejecting - `LINT_PASS` / `LOWER_BOUND` tier-swap promotions (`A7`), and catching all 8 abstract certificate + `LINT_PASS` / `LOWER_BOUND` tier-swap promotions (`A7`), requiring a KNOWN `citation` + (`source`, `reference`, `artifact_sha256`), and catching all 8 abstract certificate unsoundness classes (`F-01` through `F-08`). """ @@ -636,13 +637,90 @@ def audit_portfolio_bounds( } +def _known_citation_findings(cert: Mapping[str, Any]) -> list[dict[str, str]]: + """Require a KNOWN citation. Proof digests and `tcb_pins` are not one. + + `cert["citation"]` is `{"source", "reference", "artifact_sha256"}`. + `source` and `reference` are strings that still contain text after stripping + whitespace. `artifact_sha256` is the cited artifact and matches `^[0-9a-f]{64}$`. + """ + citation = cert.get("citation") + if not isinstance(citation, Mapping): + return [ + { + "rule": "CERT_KNOWN_CITATION_MISSING", + "message": ( + "KNOWN requires citation.source, citation.reference, and " + "citation.artifact_sha256. witness_or_proof_sha256 and tcb_pins are not a citation." + ), + } + ] + findings: list[dict[str, str]] = [] + for key in ("source", "reference"): + if key not in citation or citation.get(key) is None: + findings.append( + { + "rule": "CERT_KNOWN_CITATION_MISSING", + "message": f"KNOWN citation.{key} is absent.", + } + ) + continue + value = citation.get(key) + if not isinstance(value, str): + findings.append( + { + "rule": "CERT_KNOWN_CITATION_MALFORMED", + "message": f"KNOWN citation.{key} must be a non-empty string.", + } + ) + elif not value.strip(): + findings.append( + { + "rule": "CERT_KNOWN_CITATION_MISSING", + "message": f"KNOWN citation.{key} is blank.", + } + ) + if "artifact_sha256" not in citation or citation.get("artifact_sha256") is None: + findings.append( + { + "rule": "CERT_KNOWN_CITATION_MISSING", + "message": "KNOWN citation.artifact_sha256 is absent.", + } + ) + else: + digest = citation.get("artifact_sha256") + if not isinstance(digest, str) or not digest.strip(): + findings.append( + { + "rule": "CERT_KNOWN_CITATION_MISSING" if isinstance(digest, str) else "CERT_KNOWN_CITATION_MALFORMED", + "message": ( + "KNOWN citation.artifact_sha256 is blank." + if isinstance(digest, str) + else "KNOWN citation.artifact_sha256 must match ^[0-9a-f]{64}$." + ), + } + ) + elif not SHA256_64_RE.fullmatch(digest): + findings.append( + { + "rule": "CERT_KNOWN_CITATION_MALFORMED", + "message": f"KNOWN citation.artifact_sha256 must match ^[0-9a-f]{64}$ (got {digest!r}).", + } + ) + return findings + + def audit_transition_certificate( cert: Mapping[str, Any], *, require_must_fail_controls: bool = False, registered_artifact_digests: frozenset[str] | None = None, ) -> dict[str, Any]: - """Domain-general fail-closed auditor covering `F-01..F-08` plus `A6`/`A7` control and tier-swap gates.""" + """Domain-general fail-closed auditor covering `F-01..F-08` plus `A6`/`A7` and KNOWN citations. + + A KNOWN certificate carries `citation`: `source` and `reference` are non-empty + strings, and `artifact_sha256` is a lowercase 64-hex digest of the cited artifact. + """ findings: list[dict[str, str]] = [] status = str(cert.get("epistemic_status", "")).strip().upper() proof_mode = str(cert.get("proof_mode", "")).strip().lower() @@ -666,7 +744,7 @@ def audit_transition_certificate( evidence_kind = str(cert.get("evidence_kind", "")).strip().lower() # A7: Reject tier-swap promotions of LINT_PASS / LOWER_BOUND / bounded-subgraph - # to VERIFIED, PROVEN, or KNOWN. KNOWN uses this same evidence check. + # to VERIFIED, PROVEN, or KNOWN. A citation does not make a lower prior legal. if status in ("VERIFIED", "PROVEN", "KNOWN"): if prior_status in NON_VERIFIED_SUBTIERS or evidence_kind in ("static_lint", "lower_bound_only", "bounded_subgraph"): findings.append( @@ -683,6 +761,9 @@ def audit_transition_certificate( } ) + if status == "KNOWN": + findings.extend(_known_citation_findings(cert)) + # A6: Non-empty, distinct, non-tautological must_fail_controls if require_must_fail_controls or "must_fail_controls" in cert or "control_records" in cert: controls = cert.get("must_fail_controls") diff --git a/tests/test_audit_status_fail_closed.py b/tests/test_audit_status_fail_closed.py index 9228a3a..70e9aa9 100644 --- a/tests/test_audit_status_fail_closed.py +++ b/tests/test_audit_status_fail_closed.py @@ -13,7 +13,7 @@ from pathlib import Path from mcproof.cli import main as cli_main -from mcproof.domain_protocol import audit_transition_certificate +from mcproof.domain_protocol import RECOGNIZED_CERT_STATUSES, audit_transition_certificate from helpers import expect_eq, expect_false, expect_in, expect_true @@ -39,14 +39,35 @@ def test_unknown_status_is_rejected() -> None: expect_in("CERT_STATUS_UNKNOWN", _rules(cert)) +def _cited_known(**citation_overrides: object) -> dict: + citation = { + "source": "Euclid, Elements", + "reference": "Book I, Proposition 47", + "artifact_sha256": HEX64, + } + citation.update(citation_overrides) + return {"epistemic_status": "KNOWN", "citation": citation} + + +def _expect_citation_rejected(cert: dict, rule: str) -> None: + result = audit_transition_certificate(cert) + expect_false(result["passed"], f"KNOWN was accepted: {result}") + expect_eq(result["epistemic_status"], "REJECTED") + expect_in(rule, _rules(cert)) + + def test_known_refused_and_in_progress_are_recognized() -> None: - for tier in ("KNOWN", "REFUSED", "IN_PROGRESS"): + expect_true({"KNOWN", "REFUSED", "IN_PROGRESS"} <= RECOGNIZED_CERT_STATUSES) + for tier in ("REFUSED", "IN_PROGRESS"): cert = {"epistemic_status": tier} result = audit_transition_certificate(cert) expect_true(result["passed"], f"{tier} certificate rejected: {result}") expect_eq(result["epistemic_status"], tier) rules = _rules(cert) expect_false("CERT_STATUS_UNKNOWN" in rules, f"unexpected CERT_STATUS_UNKNOWN for {tier}: {rules}") + known_rules = _rules({"epistemic_status": "KNOWN"}) + expect_false("CERT_STATUS_UNKNOWN" in known_rules, known_rules) + expect_in("CERT_KNOWN_CITATION_MISSING", known_rules) def test_rejected_and_conjecture_are_not_relabeled_verified() -> None: @@ -60,9 +81,9 @@ def test_rejected_and_conjecture_are_not_relabeled_verified() -> None: def test_conjecture_to_known_without_evidence_is_rejected() -> None: - """A7 rejects a lower tier promoted to KNOWN with no replayed witness. + """A7 rejects a lower tier promoted to KNOWN, including one that carries a citation. - Reverting the KNOWN arm of the tier check lets this certificate pass. + Reverting the KNOWN arm of the tier check lets the cited certificate pass. """ cert = {"epistemic_status": "KNOWN", "prior_status": "CONJECTURE"} result = audit_transition_certificate(cert) @@ -70,18 +91,85 @@ def test_conjecture_to_known_without_evidence_is_rejected() -> None: expect_eq(result["epistemic_status"], "REJECTED") expect_in("CERT_A7_ILLEGAL_TIER_SWAP", _rules(cert)) - -def test_known_with_accepted_evidence_keeps_known() -> None: - """KNOWN carrying the witness pins A7 already accepts for VERIFIED stays KNOWN.""" + cited = _cited_known() + cited["prior_status"] = "CONJECTURE" + cited_result = audit_transition_certificate(cited) + expect_false(cited_result["passed"], f"cited CONJECTURE to KNOWN was accepted: {cited_result}") + expect_eq(cited_result["epistemic_status"], "REJECTED") + expect_in("CERT_A7_ILLEGAL_TIER_SWAP", _rules(cited)) + + +def test_bare_known_is_rejected() -> None: + """A bare KNOWN certificate has no citation. Removing the citation check lets it pass.""" + _expect_citation_rejected({"epistemic_status": "KNOWN"}, "CERT_KNOWN_CITATION_MISSING") + + +def test_known_missing_or_blank_source_is_rejected() -> None: + """Removing the citation check lets a citation with no usable source pass.""" + missing = _cited_known() + del missing["citation"]["source"] + _expect_citation_rejected(missing, "CERT_KNOWN_CITATION_MISSING") + _expect_citation_rejected(_cited_known(source=" "), "CERT_KNOWN_CITATION_MISSING") + _expect_citation_rejected(_cited_known(source=""), "CERT_KNOWN_CITATION_MISSING") + + +def test_known_missing_or_blank_reference_is_rejected() -> None: + """Removing the citation check lets a citation with no usable reference pass.""" + missing = _cited_known() + del missing["citation"]["reference"] + _expect_citation_rejected(missing, "CERT_KNOWN_CITATION_MISSING") + _expect_citation_rejected(_cited_known(reference=" \t"), "CERT_KNOWN_CITATION_MISSING") + + +def test_known_missing_or_malformed_digest_is_rejected() -> None: + """Removing the citation check lets a missing, short, or uppercase digest pass.""" + missing = _cited_known() + del missing["citation"]["artifact_sha256"] + _expect_citation_rejected(missing, "CERT_KNOWN_CITATION_MISSING") + _expect_citation_rejected(_cited_known(artifact_sha256="zz"), "CERT_KNOWN_CITATION_MALFORMED") + _expect_citation_rejected(_cited_known(artifact_sha256="A" * 64), "CERT_KNOWN_CITATION_MALFORMED") + + +def test_known_without_citation_is_rejected_for_other_priors() -> None: + """evidence_kind, a blank prior, and REFUSED are not a citation.""" + _expect_citation_rejected( + {"epistemic_status": "KNOWN", "evidence_kind": "none"}, + "CERT_KNOWN_CITATION_MISSING", + ) + _expect_citation_rejected( + {"epistemic_status": "KNOWN", "prior_status": " "}, + "CERT_KNOWN_CITATION_MISSING", + ) + _expect_citation_rejected( + {"epistemic_status": "KNOWN", "prior_status": "REFUSED"}, + "CERT_KNOWN_CITATION_MISSING", + ) + + +def test_known_proof_digest_and_tcb_pins_are_not_a_citation() -> None: + """The pins that satisfy VERIFIED do not satisfy KNOWN. + + Removing the citation check lets this certificate pass and stay KNOWN. + """ cert = { "epistemic_status": "KNOWN", "witness_or_proof_sha256": HEX64, "tcb_pins": {"tcb_manifest_verified": True, "checker_sha256": HEX64}, } + _expect_citation_rejected(cert, "CERT_KNOWN_CITATION_MISSING") + expect_false("CERT_A7_ILLEGAL_TIER_SWAP" in _rules(cert), _rules(cert)) + + +def test_known_with_citation_keeps_known() -> None: + """A source, a reference, and a lowercase artifact digest stay KNOWN.""" + cert = _cited_known() result = audit_transition_certificate(cert) - expect_true(result["passed"], f"KNOWN with witness evidence rejected: {result}") + expect_true(result["passed"], f"cited KNOWN rejected: {result}") expect_eq(result["epistemic_status"], "KNOWN") - expect_false("CERT_A7_ILLEGAL_TIER_SWAP" in _rules(cert), result) + rules = _rules(cert) + expect_false("CERT_KNOWN_CITATION_MISSING" in rules, result) + expect_false("CERT_KNOWN_CITATION_MALFORMED" in rules, result) + expect_false("CERT_A7_ILLEGAL_TIER_SWAP" in rules, result) def test_verified_with_pins_still_passes() -> None: From f0e7d271983c4fb0c24db05e2d1397ec5d542bec Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 20 Nov 2026 11:21:38 +0000 Subject: [PATCH 9/9] Reject invisible KNOWN citation text and an all-zero digest. source and reference must contain a printable character, and a 64-zero artifact_sha256 is malformed. A REJECTED or REFUSED prior with a valid citation is still the intended promotion to KNOWN. Co-authored-by: ahraz --- src/mcproof/domain_protocol.py | 26 ++++++++++++++++++++------ tests/test_audit_status_fail_closed.py | 18 ++++++++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/mcproof/domain_protocol.py b/src/mcproof/domain_protocol.py index a6111b0..ad42143 100644 --- a/src/mcproof/domain_protocol.py +++ b/src/mcproof/domain_protocol.py @@ -42,6 +42,8 @@ MAX_CERT_ABS_INT = 10**60 STRICT_RATIONAL_RE = re.compile(r"^-?[0-9]+(?:/[1-9][0-9]*)?$") SHA256_64_RE = re.compile(r"^[0-9a-f]{64}$") +# Zero-width and word-joiner marks. Unicode whitespace is rejected separately. +_CITATION_INVISIBLE_CHARS = frozenset("\u200b\u200c\u200d\u2060\ufeff") NON_VERIFIED_SUBTIERS: frozenset[str] = frozenset( { "LINT_PASS", @@ -637,12 +639,21 @@ def audit_portfolio_bounds( } +def _citation_text_visible(value: str) -> bool: + """True when `value` has a printable character that is not whitespace or invisible.""" + return any( + ch.isprintable() and not ch.isspace() and ch not in _CITATION_INVISIBLE_CHARS + for ch in value + ) + + def _known_citation_findings(cert: Mapping[str, Any]) -> list[dict[str, str]]: """Require a KNOWN citation. Proof digests and `tcb_pins` are not one. `cert["citation"]` is `{"source", "reference", "artifact_sha256"}`. - `source` and `reference` are strings that still contain text after stripping - whitespace. `artifact_sha256` is the cited artifact and matches `^[0-9a-f]{64}$`. + `source` and `reference` need at least one printable, non-invisible character. + `artifact_sha256` matches `^[0-9a-f]{64}$` and is not 64 zeros. + A REJECTED or REFUSED prior plus a valid citation is intended to promote to KNOWN. """ citation = cert.get("citation") if not isinstance(citation, Mapping): @@ -673,11 +684,11 @@ def _known_citation_findings(cert: Mapping[str, Any]) -> list[dict[str, str]]: "message": f"KNOWN citation.{key} must be a non-empty string.", } ) - elif not value.strip(): + elif not _citation_text_visible(value): findings.append( { "rule": "CERT_KNOWN_CITATION_MISSING", - "message": f"KNOWN citation.{key} is blank.", + "message": f"KNOWN citation.{key} is blank or has no visible character.", } ) if "artifact_sha256" not in citation or citation.get("artifact_sha256") is None: @@ -700,11 +711,14 @@ def _known_citation_findings(cert: Mapping[str, Any]) -> list[dict[str, str]]: ), } ) - elif not SHA256_64_RE.fullmatch(digest): + elif not SHA256_64_RE.fullmatch(digest) or digest == "0" * 64: findings.append( { "rule": "CERT_KNOWN_CITATION_MALFORMED", - "message": f"KNOWN citation.artifact_sha256 must match ^[0-9a-f]{64}$ (got {digest!r}).", + "message": ( + "KNOWN citation.artifact_sha256 must match ^[0-9a-f]{64}$ " + f"and must not be 64 zeros (got {digest!r})." + ), } ) return findings diff --git a/tests/test_audit_status_fail_closed.py b/tests/test_audit_status_fail_closed.py index 70e9aa9..cdb1158 100644 --- a/tests/test_audit_status_fail_closed.py +++ b/tests/test_audit_status_fail_closed.py @@ -160,6 +160,24 @@ def test_known_proof_digest_and_tcb_pins_are_not_a_citation() -> None: expect_false("CERT_A7_ILLEGAL_TIER_SWAP" in _rules(cert), _rules(cert)) +def test_known_invisible_citation_text_is_rejected() -> None: + """Invisible source or reference text is missing. + + Disabling the visible-character check lets these certificates pass. + """ + _expect_citation_rejected(_cited_known(source="\u200b"), "CERT_KNOWN_CITATION_MISSING") + _expect_citation_rejected(_cited_known(reference="\ufeff "), "CERT_KNOWN_CITATION_MISSING") + _expect_citation_rejected(_cited_known(source="\u00a0\u2060"), "CERT_KNOWN_CITATION_MISSING") + + +def test_known_all_zero_artifact_digest_is_rejected() -> None: + """Sixty-four zeros match the hex pattern and are still malformed.""" + _expect_citation_rejected( + _cited_known(artifact_sha256="0" * 64), + "CERT_KNOWN_CITATION_MALFORMED", + ) + + def test_known_with_citation_keeps_known() -> None: """A source, a reference, and a lowercase artifact digest stay KNOWN.""" cert = _cited_known()