From f39df449675e8246a49745e44549b553e6a8a4b0 Mon Sep 17 00:00:00 2001 From: Renato Date: Mon, 17 Aug 2026 07:07:27 +0200 Subject: [PATCH] fix: harden receipts and benchmark claims --- README.md | 5 ++ benchmark/analysis/summary.md | 4 +- benchmark/results/README.md | 6 ++ benchmarks/swebench_lite/PUBLIC_BENCHMARK.md | 24 ++----- .../PUBLIC_BENCHMARK.md | 8 +-- .../baseline_metrics.ndjson | 6 +- .../baseline_verified.ndjson | 6 +- .../marginal_metrics.ndjson | 6 +- .../marginal_verified.ndjson | 6 +- .../public-benchmark.json | 2 +- .../swebench_lite/public-benchmark.json | 2 +- plugins/marginal/runtime/marginal_runtime.pyz | Bin 523971 -> 525623 bytes plugins/marginal/runtime/provenance.json | 2 +- site/benchmark-data.json | 2 +- src/marginal/public_eval.py | 63 ++++++++++++------ src/marginal/receipts.py | 3 + tests/test_public_eval_v2.py | 19 ++++++ tests/test_receipts.py | 11 +++ 18 files changed, 115 insertions(+), 60 deletions(-) create mode 100644 benchmark/results/README.md diff --git a/README.md b/README.md index 922aba4..45e1084 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,11 @@ explicit promotion. Authority is contextual, not permanent. The Trust Engine evaluates sample size, coverage, harmful outcomes, regret, governance tax, recency, policy identity, and available adapter capabilities. +The promotion gate assumes a non-adversarial, same-user trust domain. Receipt and ledger hashes +detect corruption and make decisions reproducible; they do not prove authorship or stop the agent, +a prompt-injected tool, or another same-user process from rewriting local state. Do not treat local +Earned Enforcement as a security boundary against software running as the same OS user. + ## User intent and controls Codex user prompts can express deliberate repeat intent in English or Italian, including `repeat`, diff --git a/benchmark/analysis/summary.md b/benchmark/analysis/summary.md index 2104f5a..405fa0e 100644 --- a/benchmark/analysis/summary.md +++ b/benchmark/analysis/summary.md @@ -2,7 +2,9 @@ ## Executive Summary -No Codex OFF/ON trajectory was executed. The integration-only preflight passes for the +This directory records an earlier blocked preflight; no Codex OFF/ON trajectory was executed in +that attempt. A later exploratory smoke did run, and its complete evidence is published under +`benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/`. The integration-only preflight passes for the frozen three-task set, Codex 0.147.0, `gpt-5.6-sol`, and `high` reasoning effort. The mandatory full preflight stops before inference because successful shell executions are not observable through the pinned hook contract and no official per-instance SWE-bench diff --git a/benchmark/results/README.md b/benchmark/results/README.md new file mode 100644 index 0000000..ea6e567 --- /dev/null +++ b/benchmark/results/README.md @@ -0,0 +1,6 @@ +# Benchmark results + +These empty files belong to the earlier blocked preflight and contain no Codex trajectories. The +completed exploratory smoke evidence is under +[`benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/`](../../benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/), +including paired telemetry, verifier reports, provenance, verification, and the public report. diff --git a/benchmarks/swebench_lite/PUBLIC_BENCHMARK.md b/benchmarks/swebench_lite/PUBLIC_BENCHMARK.md index d524e5b..8f88d88 100644 --- a/benchmarks/swebench_lite/PUBLIC_BENCHMARK.md +++ b/benchmarks/swebench_lite/PUBLIC_BENCHMARK.md @@ -8,7 +8,7 @@ MARGINAL overhead is counted in net efficiency and net savings. | Resolved | 0/3 | 0/3 | +0.00 pp | | Agent tokens | 1,098,747 | 824,839 | 24.93% fewer | | Effective tokens (incl. governance) | 1,098,747 | 824,839 | 24.93% fewer | -| Effective USD | $0.0000 | $0.0000 | 0.00% lower | +| Effective USD | n/a | n/a | n/a | | Effective latency | 593,106 ms | 565,768 ms | 4.61% lower | | Tool calls | 33 | 32 | 3.03% fewer | | Repeated calls | 0 | 0 | 0.00% fewer | @@ -17,32 +17,16 @@ MARGINAL overhead is counted in net efficiency and net savings. ## Governance tax -MARGINAL overhead: **0 tokens**, **$0.000000**, **7,058 ms**. +MARGINAL overhead: **0 tokens**, **n/a**, **7,058 ms**. Gross agent-token savings: **24.93%**. Net token savings after governance: **24.93%**. ## Quality and intervention decision -Net token savings 95.0% bootstrap interval: **21.52% to 28.07%**. -Quality preserved within the 1.00 pp non-inferiority margin: **True**. +Token uncertainty: **not evaluable** without a successful task in both arms. +Quality preserved within the 1.00 pp non-inferiority margin: **not evaluable**. Regressions: **0**. Recoveries: **0**. Reviewed deny recommendations: **0**. False stops: **0** (n/a). Intervention status: **pass_through**. No verified successful task was observed, so token efficiency per resolved task is undefined and the intervention cannot be classified as supported. `pass_through` is a valid result: it means MARGINAL did not demonstrate enough net value to justify intervention under the preregistered threshold. - -## Scope and interpretation - -This is an **exploratory 3-task smoke with one paired run per task**, not a population estimate or headline performance claim. Codex CLI 0.147.0 and GPT-5.6-sol ran under identical prompts, limits, task order, official task images and base commits. MARGINAL added 7,058 ms of measured local governance latency, zero external governance tokens and zero governance USD. - -The token interval above is only a bootstrap over three task pairs; it cannot estimate run-to-run model variance because there was one repetition. All three observed ON trajectories had zero applied denies. The token difference therefore cannot be attributed to a stop decision, and neither lane's 0/3 resolve rate permits an efficiency-per-success claim. - -## Verification chain - -- Authoritative correctness: SWE-bench 4.1.0 Docker harness, x86_64, official pinned task images; 3/3 completed and 0 infrastructure errors in each lane. -- Cloud audit: [GitHub Actions + Modal run 31474500980](https://github.com/SignalLayerLabs/Marginal/actions/runs/31474500980); 2/3 completed and the same Modal image-build error occurred for `pylint-dev__astroid-1978` in both lanes. The cloud error is retained but excluded from scoring. -- Evidence: [`evidence/smoke-2026-08-11-dbce533/`](evidence/smoke-2026-08-11-dbce533/) contains predictions, telemetry, verifier reports, merged rows, provenance and raw public-evaluator output. -- Frozen task-set SHA-256: `1c8c06935484fd57f4c27c363418884542bcfb06c69d02f480b61fe7687b58eb`. -- MARGINAL source commit used by the runtime overlays: `dbce533e5c8e8def62b0543853d07cfab2bc79c1`. - -The next promotion gate is a preregistered repeated canary with verified successes and enough repetitions to measure trajectory variance. Until then, `pass_through` is the evidence-backed product decision. diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/PUBLIC_BENCHMARK.md b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/PUBLIC_BENCHMARK.md index ed56415..8f88d88 100644 --- a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/PUBLIC_BENCHMARK.md +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/PUBLIC_BENCHMARK.md @@ -8,7 +8,7 @@ MARGINAL overhead is counted in net efficiency and net savings. | Resolved | 0/3 | 0/3 | +0.00 pp | | Agent tokens | 1,098,747 | 824,839 | 24.93% fewer | | Effective tokens (incl. governance) | 1,098,747 | 824,839 | 24.93% fewer | -| Effective USD | $0.0000 | $0.0000 | 0.00% lower | +| Effective USD | n/a | n/a | n/a | | Effective latency | 593,106 ms | 565,768 ms | 4.61% lower | | Tool calls | 33 | 32 | 3.03% fewer | | Repeated calls | 0 | 0 | 0.00% fewer | @@ -17,13 +17,13 @@ MARGINAL overhead is counted in net efficiency and net savings. ## Governance tax -MARGINAL overhead: **0 tokens**, **$0.000000**, **7,058 ms**. +MARGINAL overhead: **0 tokens**, **n/a**, **7,058 ms**. Gross agent-token savings: **24.93%**. Net token savings after governance: **24.93%**. ## Quality and intervention decision -Net token savings 95.0% bootstrap interval: **21.52% to 28.07%**. -Quality preserved within the 1.00 pp non-inferiority margin: **True**. +Token uncertainty: **not evaluable** without a successful task in both arms. +Quality preserved within the 1.00 pp non-inferiority margin: **not evaluable**. Regressions: **0**. Recoveries: **0**. Reviewed deny recommendations: **0**. False stops: **0** (n/a). Intervention status: **pass_through**. diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_metrics.ndjson b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_metrics.ndjson index 75a79b3..a3ce752 100644 --- a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_metrics.ndjson +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_metrics.ndjson @@ -1,3 +1,3 @@ -{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1072","latency_ms":177424,"repeated_calls":0,"reviewed_stops":0,"tokens":370497,"tool_calls":11,"usd":0.0} -{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1707","latency_ms":268283,"repeated_calls":0,"reviewed_stops":0,"tokens":383851,"tool_calls":12,"usd":0.0} -{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pylint-dev__astroid-1978","latency_ms":147399,"repeated_calls":0,"reviewed_stops":0,"tokens":344399,"tool_calls":10,"usd":0.0} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1072","latency_ms":177424,"repeated_calls":0,"reviewed_stops":0,"tokens":370497,"tool_calls":11,"usd":0.0,"usd_measured":false} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1707","latency_ms":268283,"repeated_calls":0,"reviewed_stops":0,"tokens":383851,"tool_calls":12,"usd":0.0,"usd_measured":false} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pylint-dev__astroid-1978","latency_ms":147399,"repeated_calls":0,"reviewed_stops":0,"tokens":344399,"tool_calls":10,"usd":0.0,"usd_measured":false} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_verified.ndjson b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_verified.ndjson index 6e43e68..8211771 100644 --- a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_verified.ndjson +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/baseline_verified.ndjson @@ -1,3 +1,3 @@ -{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1072","latency_ms":177424,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":370497,"tool_calls":11,"usd":0.0} -{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1707","latency_ms":268283,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":383851,"tool_calls":12,"usd":0.0} -{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pylint-dev__astroid-1978","latency_ms":147399,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":344399,"tool_calls":10,"usd":0.0} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1072","latency_ms":177424,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":370497,"tool_calls":11,"usd":0.0,"usd_measured":false} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1707","latency_ms":268283,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":383851,"tool_calls":12,"usd":0.0,"usd_measured":false} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":0,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pylint-dev__astroid-1978","latency_ms":147399,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":344399,"tool_calls":10,"usd":0.0,"usd_measured":false} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_metrics.ndjson b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_metrics.ndjson index d321ce7..5af0cc9 100644 --- a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_metrics.ndjson +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_metrics.ndjson @@ -1,3 +1,3 @@ -{"applied_denies":0,"false_stops":0,"governance_latency_ms":1823,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1072","latency_ms":178765,"repeated_calls":0,"reviewed_stops":0,"tokens":275858,"tool_calls":11,"usd":0.0} -{"applied_denies":0,"false_stops":0,"governance_latency_ms":2224,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1707","latency_ms":236330,"repeated_calls":0,"reviewed_stops":0,"tokens":301257,"tool_calls":13,"usd":0.0} -{"applied_denies":0,"false_stops":0,"governance_latency_ms":3011,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pylint-dev__astroid-1978","latency_ms":143615,"repeated_calls":0,"reviewed_stops":0,"tokens":247724,"tool_calls":8,"usd":0.0} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":1823,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1072","latency_ms":178765,"repeated_calls":0,"reviewed_stops":0,"tokens":275858,"tool_calls":11,"usd":0.0,"usd_measured":false} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":2224,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1707","latency_ms":236330,"repeated_calls":0,"reviewed_stops":0,"tokens":301257,"tool_calls":13,"usd":0.0,"usd_measured":false} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":3011,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pylint-dev__astroid-1978","latency_ms":143615,"repeated_calls":0,"reviewed_stops":0,"tokens":247724,"tool_calls":8,"usd":0.0,"usd_measured":false} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_verified.ndjson b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_verified.ndjson index 9b93096..5bb4192 100644 --- a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_verified.ndjson +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/marginal_verified.ndjson @@ -1,3 +1,3 @@ -{"applied_denies":0,"false_stops":0,"governance_latency_ms":1823,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1072","latency_ms":178765,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":275858,"tool_calls":11,"usd":0.0} -{"applied_denies":0,"false_stops":0,"governance_latency_ms":2224,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1707","latency_ms":236330,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":301257,"tool_calls":13,"usd":0.0} -{"applied_denies":0,"false_stops":0,"governance_latency_ms":3011,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pylint-dev__astroid-1978","latency_ms":143615,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":247724,"tool_calls":8,"usd":0.0} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":1823,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1072","latency_ms":178765,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":275858,"tool_calls":11,"usd":0.0,"usd_measured":false} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":2224,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pvlib__pvlib-python-1707","latency_ms":236330,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":301257,"tool_calls":13,"usd":0.0,"usd_measured":false} +{"applied_denies":0,"false_stops":0,"governance_latency_ms":3011,"governance_tokens":0,"governance_usd":0.0,"instance_id":"pylint-dev__astroid-1978","latency_ms":143615,"repeated_calls":0,"resolved":false,"reviewed_stops":0,"tokens":247724,"tool_calls":8,"usd":0.0,"usd_measured":false} diff --git a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/public-benchmark.json b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/public-benchmark.json index 8c048dd..7fbd34e 100644 --- a/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/public-benchmark.json +++ b/benchmarks/swebench_lite/evidence/smoke-2026-08-11-dbce533/public-benchmark.json @@ -1 +1 @@ -{"agent_only_efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "baseline": {"effective_latency_ms": 593106, "effective_tokens": 1098747, "effective_usd": 0.0, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 0, "governance_tokens": 0, "governance_usd": 0.0, "latency_ms": 593106, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 1098747, "tool_calls": 33, "usd": 0.0}, "benchmark": "public-agent-benchmark-comparison-v3", "efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "governance": {"latency_ms": 7058, "tokens": 0, "usd": 0.0}, "gross_savings": {"confidence_level": 0.95, "latency_percent": 5.8, "repeated_calls_percent": 0.0, "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "intervention": {"eligible_for_support": false, "graceful_irrelevance": true, "minimum_net_token_savings_percent": 0.0, "net_positive": true, "status": "pass_through"}, "marginal": {"effective_latency_ms": 565768, "effective_tokens": 824839, "effective_usd": 0.0, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 7058, "governance_tokens": 0, "governance_usd": 0.0, "latency_ms": 558710, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 824839, "tool_calls": 32, "usd": 0.0}, "net_savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": [21.52, 28.07], "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "quality": {"false_stop_rate": null, "false_stops": 0, "false_stops_acceptable": true, "has_verified_success": false, "max_false_stop_rate": 0.0, "non_inferiority_margin_pp": 1.0, "preserved_within_margin": true, "preserved_within_one_pp": true, "recoveries": 0, "regressions": 0, "resolved_delta_pp": 0.0, "reviewed_stops": 0}, "savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": [21.52, 28.07], "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "tasks": 3} +{"agent_only_efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "baseline": {"effective_latency_ms": 593106, "effective_tokens": 1098747, "effective_usd": null, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 0, "governance_tokens": 0, "governance_usd": null, "latency_ms": 593106, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 1098747, "tool_calls": 33, "usd": null}, "benchmark": "public-agent-benchmark-comparison-v3", "efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "governance": {"latency_ms": 7058, "tokens": 0, "usd": null}, "gross_savings": {"confidence_level": 0.95, "latency_percent": 5.8, "repeated_calls_percent": 0.0, "tokens_confidence_interval": null, "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": null}, "intervention": {"eligible_for_support": false, "graceful_irrelevance": true, "minimum_net_token_savings_percent": 0.0, "net_positive": true, "status": "pass_through"}, "marginal": {"effective_latency_ms": 565768, "effective_tokens": 824839, "effective_usd": null, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 7058, "governance_tokens": 0, "governance_usd": null, "latency_ms": 558710, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 824839, "tool_calls": 32, "usd": null}, "net_savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": null, "tokens_confidence_interval": null, "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": null}, "quality": {"evaluable": false, "false_stop_rate": null, "false_stops": 0, "false_stops_acceptable": true, "has_verified_success": false, "max_false_stop_rate": 0.0, "non_inferiority_margin_pp": 1.0, "preserved_within_margin": null, "preserved_within_one_pp": null, "recoveries": 0, "regressions": 0, "resolved_delta_pp": 0.0, "reviewed_stops": 0}, "savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": null, "tokens_confidence_interval": null, "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": null}, "tasks": 3} diff --git a/benchmarks/swebench_lite/public-benchmark.json b/benchmarks/swebench_lite/public-benchmark.json index 8c048dd..7fbd34e 100644 --- a/benchmarks/swebench_lite/public-benchmark.json +++ b/benchmarks/swebench_lite/public-benchmark.json @@ -1 +1 @@ -{"agent_only_efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "baseline": {"effective_latency_ms": 593106, "effective_tokens": 1098747, "effective_usd": 0.0, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 0, "governance_tokens": 0, "governance_usd": 0.0, "latency_ms": 593106, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 1098747, "tool_calls": 33, "usd": 0.0}, "benchmark": "public-agent-benchmark-comparison-v3", "efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "governance": {"latency_ms": 7058, "tokens": 0, "usd": 0.0}, "gross_savings": {"confidence_level": 0.95, "latency_percent": 5.8, "repeated_calls_percent": 0.0, "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "intervention": {"eligible_for_support": false, "graceful_irrelevance": true, "minimum_net_token_savings_percent": 0.0, "net_positive": true, "status": "pass_through"}, "marginal": {"effective_latency_ms": 565768, "effective_tokens": 824839, "effective_usd": 0.0, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 7058, "governance_tokens": 0, "governance_usd": 0.0, "latency_ms": 558710, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 824839, "tool_calls": 32, "usd": 0.0}, "net_savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": [21.52, 28.07], "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "quality": {"false_stop_rate": null, "false_stops": 0, "false_stops_acceptable": true, "has_verified_success": false, "max_false_stop_rate": 0.0, "non_inferiority_margin_pp": 1.0, "preserved_within_margin": true, "preserved_within_one_pp": true, "recoveries": 0, "regressions": 0, "resolved_delta_pp": 0.0, "reviewed_stops": 0}, "savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": [21.52, 28.07], "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "tasks": 3} +{"agent_only_efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "baseline": {"effective_latency_ms": 593106, "effective_tokens": 1098747, "effective_usd": null, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 0, "governance_tokens": 0, "governance_usd": null, "latency_ms": 593106, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 1098747, "tool_calls": 33, "usd": null}, "benchmark": "public-agent-benchmark-comparison-v3", "efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "governance": {"latency_ms": 7058, "tokens": 0, "usd": null}, "gross_savings": {"confidence_level": 0.95, "latency_percent": 5.8, "repeated_calls_percent": 0.0, "tokens_confidence_interval": null, "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": null}, "intervention": {"eligible_for_support": false, "graceful_irrelevance": true, "minimum_net_token_savings_percent": 0.0, "net_positive": true, "status": "pass_through"}, "marginal": {"effective_latency_ms": 565768, "effective_tokens": 824839, "effective_usd": null, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 7058, "governance_tokens": 0, "governance_usd": null, "latency_ms": 558710, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 824839, "tool_calls": 32, "usd": null}, "net_savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": null, "tokens_confidence_interval": null, "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": null}, "quality": {"evaluable": false, "false_stop_rate": null, "false_stops": 0, "false_stops_acceptable": true, "has_verified_success": false, "max_false_stop_rate": 0.0, "non_inferiority_margin_pp": 1.0, "preserved_within_margin": null, "preserved_within_one_pp": null, "recoveries": 0, "regressions": 0, "resolved_delta_pp": 0.0, "reviewed_stops": 0}, "savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": null, "tokens_confidence_interval": null, "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": null}, "tasks": 3} diff --git a/plugins/marginal/runtime/marginal_runtime.pyz b/plugins/marginal/runtime/marginal_runtime.pyz index 2118d4ba4b65a5f0937317a083930ce9bd36b41f..750ced97341008355c1dbcf4f0e0facb9273b413 100644 GIT binary patch delta 2080 zcmah~eQXnD7+m@qae|3e2uWpQf}>aoV(qzyX(xr z1r}#8B*+x?G0K7vq9zjjA{@pAH70^ajr&6+AyNL&Xw(Tt zTRxRyG+jqUSr=)Ys)zv>9zIWUUQ_&F^ZvIP?tTbz@%>`R!u<>2ssFJwJpp*qwou(} zNU9>pVO{8yl-3eelUgMvm(dkUSAbgTc4>#jCzG-NN8+)Pr>v-?C#qw>q$w7abNqdp zXv8&zrsBqARWK%^`vPP-wol5DKO(?}!VsWORkaPC%Gfxg?v z8(RsBZku#r5IVEcE5nOt$ZYJXha4OT!jNl%`ZBCgIX8MA1Q!mih8**yRqzJnr2D39 zaqKteocz?x__ABuHiNbBmpzjm9&0-j^P0N~W}6M`pvm!oHJivbL20$-%`1xp-pZ0Z zpL%m*^R|mxL{tnuG;shEM8co$_)ifwj0;&3uWVsk>287xc4J>7RGQB;fy0SUZvnTvRqYftMNlGQ^L$3f{E)BP z?+@UN7Dm#EFkG==`6)sQ#ykA?nz5saxN9lDQEd|y8fURILtyLERYv{(8yfNZ>0^?y zO^q8=pn5zK5p_Klm#HD>ZEWG#O=@hTg4V9%;jJ(i-)v;3;|l>IQ^HAVyYbLgpr+gc z4&r)wX+dRiX+}nIskuXf%{IKuI{`nKYR{b;T(P0PE?iT;G8o9DEe+Da3tqoMefGg(D24R-vqfot98)ohgyV*kzN@F?`Y;KRQY%?)uts$Guu2 zX42m>*2GpxH?%IwNY&YGM|ZBjwUlmLe1!0OJbS_P>?L*WX}=d#uaXAtTpNCJm6UT% z8J*XNzhJV_jEtO&$>(U#Wd(4T#1`22m1JYbHR83e>Ib~^LJE4k&3h6#myruk!P5A zxq-o?A%g+4KZk65WSCj|&!@q^48k-#Zo#!bB;lc9Qq2tvS;9u{xj(FTJ`N4R^kn4t zl^cq?Fp~5NT!lPzjj%ZV8Mt_qmA7Vu+|RABCxwv_rhdy(KMM)u`HRGz4Cv=cdbfQg zmVPa_V}bR~!B6LrTs$_)JhPYJ`B6rCcsY8;NEKJwhmB)QZu|yc93%7K)!zvt?H?b1 B(y9Oe delta 873 zcmZ9KTWDNG7{|{!bIyNGwk8|fT$-B8A z9oF%)3Iup(GuElsL)dLA*GIp}^R3vx=6T`p!zYmA^m{1dsgMr85y6-`vK4QdoQoWQK)7Z|Y4vg3V&-1vy3#DA_z@iFwVhXBgFa8pu zH=_xiC|99w^w_-qr+Ad{B~h;4*@s^QKS-cxOLy09?-j3Zw6FJY>iMBeswbREr&Ike z4vnOex`S|{aa%+B7|lMEn{}K?>N>$B95-*)neR4bs_E@VBR@(YppGPAJC${#gTo`K z7c#w>ehvHw^Hlnn??*g!aRB#B{x*Qk93Di!)e~kiu1670X#X~T)u(58e-O3|7F>9? zAhy*oR(FYEa3&{9RqZfhrh5M%cGPfg8bQto$2mDZ0X04$pF8lMN9NDVvfJuNvoR;5 z*2`%g%}KxVe^BG!_3??mA%2;Y&DOQkG*|T5FvVRfGG=w2=Nl`s$&z35lGftXw_x=g zs(5or7EkVUH;LeP;F&o@CP!UgbX>`cjgvEOqxk&lnqcxK3VGCYi`lb);=+ejZ~c^y zDB`zw%crd@fK%S^};+KaoiAg-j%w8zX2l#5eNVP diff --git a/plugins/marginal/runtime/provenance.json b/plugins/marginal/runtime/provenance.json index e9d2c69..141cb9f 100644 --- a/plugins/marginal/runtime/provenance.json +++ b/plugins/marginal/runtime/provenance.json @@ -1 +1 @@ -{"builder":"scripts/build_codex_plugin.py","python_requires":">=3.10","schema_version":1,"sha256":"73f73a7578f15d9bec3f4a0fcc0882514d8f5c370865bf3bbd6c90d72ef4b5dd","source_hash":"c96b9c73d8cd9191af890a6aa3f71568ceb86bf155d41cd1e94eee2b23e0a6e0"} +{"builder":"scripts/build_codex_plugin.py","python_requires":">=3.10","schema_version":1,"sha256":"57b0f86a9c49daff2c0213d5606d2c369c4e39580b52cd76cfd20fdb2de33167","source_hash":"fccff1138073eb96d48fbb60470bb7f4d6d27770da7a2f61bc490b83ee3a912a"} diff --git a/site/benchmark-data.json b/site/benchmark-data.json index 8c048dd..7fbd34e 100644 --- a/site/benchmark-data.json +++ b/site/benchmark-data.json @@ -1 +1 @@ -{"agent_only_efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "baseline": {"effective_latency_ms": 593106, "effective_tokens": 1098747, "effective_usd": 0.0, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 0, "governance_tokens": 0, "governance_usd": 0.0, "latency_ms": 593106, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 1098747, "tool_calls": 33, "usd": 0.0}, "benchmark": "public-agent-benchmark-comparison-v3", "efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "governance": {"latency_ms": 7058, "tokens": 0, "usd": 0.0}, "gross_savings": {"confidence_level": 0.95, "latency_percent": 5.8, "repeated_calls_percent": 0.0, "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "intervention": {"eligible_for_support": false, "graceful_irrelevance": true, "minimum_net_token_savings_percent": 0.0, "net_positive": true, "status": "pass_through"}, "marginal": {"effective_latency_ms": 565768, "effective_tokens": 824839, "effective_usd": 0.0, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 7058, "governance_tokens": 0, "governance_usd": 0.0, "latency_ms": 558710, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 824839, "tool_calls": 32, "usd": 0.0}, "net_savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": [21.52, 28.07], "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "quality": {"false_stop_rate": null, "false_stops": 0, "false_stops_acceptable": true, "has_verified_success": false, "max_false_stop_rate": 0.0, "non_inferiority_margin_pp": 1.0, "preserved_within_margin": true, "preserved_within_one_pp": true, "recoveries": 0, "regressions": 0, "resolved_delta_pp": 0.0, "reviewed_stops": 0}, "savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": [21.52, 28.07], "tokens_confidence_interval": [21.52, 28.07], "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": 0.0}, "tasks": 3} +{"agent_only_efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "baseline": {"effective_latency_ms": 593106, "effective_tokens": 1098747, "effective_usd": null, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 0, "governance_tokens": 0, "governance_usd": null, "latency_ms": 593106, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 1098747, "tool_calls": 33, "usd": null}, "benchmark": "public-agent-benchmark-comparison-v3", "efficiency": {"baseline": {"tokens_per_resolved": null, "usd_per_resolved": null}, "marginal": {"tokens_per_resolved": null, "usd_per_resolved": null}}, "governance": {"latency_ms": 7058, "tokens": 0, "usd": null}, "gross_savings": {"confidence_level": 0.95, "latency_percent": 5.8, "repeated_calls_percent": 0.0, "tokens_confidence_interval": null, "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": null}, "intervention": {"eligible_for_support": false, "graceful_irrelevance": true, "minimum_net_token_savings_percent": 0.0, "net_positive": true, "status": "pass_through"}, "marginal": {"effective_latency_ms": 565768, "effective_tokens": 824839, "effective_usd": null, "false_stop_rate": null, "false_stops": 0, "governance_latency_ms": 7058, "governance_tokens": 0, "governance_usd": null, "latency_ms": 558710, "repeated_calls": 0, "resolve_rate": 0.0, "resolved": 0, "reviewed_stops": 0, "tasks": 3, "tokens": 824839, "tool_calls": 32, "usd": null}, "net_savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": null, "tokens_confidence_interval": null, "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": null}, "quality": {"evaluable": false, "false_stop_rate": null, "false_stops": 0, "false_stops_acceptable": true, "has_verified_success": false, "max_false_stop_rate": 0.0, "non_inferiority_margin_pp": 1.0, "preserved_within_margin": null, "preserved_within_one_pp": null, "recoveries": 0, "regressions": 0, "resolved_delta_pp": 0.0, "reviewed_stops": 0}, "savings": {"confidence_level": 0.95, "latency_percent": 4.61, "repeated_calls_percent": 0.0, "tokens_95pct_ci": null, "tokens_confidence_interval": null, "tokens_percent": 24.93, "tool_calls_percent": 3.03, "usd_percent": null}, "tasks": 3} diff --git a/src/marginal/public_eval.py b/src/marginal/public_eval.py index 2a0d4c7..f9ac0b5 100644 --- a/src/marginal/public_eval.py +++ b/src/marginal/public_eval.py @@ -28,6 +28,7 @@ class RunRecord: repeated_calls: int = 0 governance_tokens: int = 0 governance_usd: float = 0.0 + usd_measured: bool = True governance_latency_ms: int = 0 reviewed_stops: int = 0 false_stops: int = 0 @@ -59,6 +60,8 @@ def __post_init__(self) -> None: if not math.isfinite(float(value)) or value < 0: raise ValueError("metrics must be finite and non-negative") object.__setattr__(self, name, float(value)) + if not isinstance(self.usd_measured, bool): + raise TypeError("usd_measured must be a boolean") if self.false_stops > self.reviewed_stops: raise ValueError("false_stops cannot exceed reviewed_stops") @@ -97,6 +100,7 @@ def load_runs(path: Path) -> dict[str, RunRecord]: repeated_calls=item.get("repeated_calls", 0), governance_tokens=item.get("governance_tokens", 0), governance_usd=item.get("governance_usd", 0.0), + usd_measured=item.get("usd_measured", True), governance_latency_ms=item.get("governance_latency_ms", 0), reviewed_stops=item.get("reviewed_stops", 0), false_stops=item.get("false_stops", 0), @@ -119,6 +123,7 @@ def _aggregate(records: list[RunRecord]) -> dict[str, Any]: governance_latency_ms = sum(record.governance_latency_ms for record in records) tokens = sum(record.tokens for record in records) usd = sum(record.usd for record in records) + usd_measured = all(record.usd_measured for record in records) latency_ms = sum(record.latency_ms for record in records) reviewed_stops = sum(record.reviewed_stops for record in records) false_stops = sum(record.false_stops for record in records) @@ -127,15 +132,15 @@ def _aggregate(records: list[RunRecord]) -> dict[str, Any]: "resolved": resolved, "resolve_rate": resolved / tasks, "tokens": tokens, - "usd": round(usd, 6), + "usd": round(usd, 6) if usd_measured else None, "latency_ms": latency_ms, "tool_calls": sum(record.tool_calls for record in records), "repeated_calls": sum(record.repeated_calls for record in records), "governance_tokens": governance_tokens, - "governance_usd": round(governance_usd, 6), + "governance_usd": round(governance_usd, 6) if usd_measured else None, "governance_latency_ms": governance_latency_ms, "effective_tokens": tokens + governance_tokens, - "effective_usd": round(usd + governance_usd, 6), + "effective_usd": round(usd + governance_usd, 6) if usd_measured else None, "effective_latency_ms": latency_ms + governance_latency_ms, "reviewed_stops": reviewed_stops, "false_stops": false_stops, @@ -147,6 +152,12 @@ def _saving(original: float, optimized: float) -> float: return round((original - optimized) / original * 100.0, 2) if original else 0.0 +def _optional_saving(original: Any, optimized: Any) -> float | None: + if original is None or optimized is None: + return None + return _saving(float(original), float(optimized)) + + def _bootstrap_token_savings( pairs: list[tuple[RunRecord, RunRecord]], samples: int, @@ -225,6 +236,7 @@ def compare_runs( marginal_rows = [marginal[item] for item in ids] baseline_total = _aggregate(baseline_rows) marginal_total = _aggregate(marginal_rows) + quality_evaluable = int(baseline_total["resolved"]) > 0 and int(marginal_total["resolved"]) > 0 delta_pp = round( (marginal_total["resolve_rate"] - baseline_total["resolve_rate"]) * 100.0, 2, @@ -253,12 +265,14 @@ def efficiency(total: dict[str, Any], *, include_governance: bool) -> dict[str, usd_key = "effective_usd" if include_governance else "usd" return { "tokens_per_resolved": round(float(total[token_key]) / resolved, 6), - "usd_per_resolved": round(float(total[usd_key]) / resolved, 6), + "usd_per_resolved": ( + round(float(total[usd_key]) / resolved, 6) if total[usd_key] is not None else None + ), } gross_savings = { "tokens_percent": _saving(float(baseline_total["tokens"]), float(marginal_total["tokens"])), - "usd_percent": _saving(float(baseline_total["usd"]), float(marginal_total["usd"])), + "usd_percent": _optional_saving(baseline_total["usd"], marginal_total["usd"]), "latency_percent": _saving( float(baseline_total["latency_ms"]), float(marginal_total["latency_ms"]), @@ -272,16 +286,15 @@ def efficiency(total: dict[str, Any], *, include_governance: bool) -> dict[str, float(marginal_total["repeated_calls"]), ), "confidence_level": confidence_level, - "tokens_confidence_interval": list(gross_token_ci), + "tokens_confidence_interval": list(gross_token_ci) if quality_evaluable else None, } net_savings = { "tokens_percent": _saving( float(baseline_total["effective_tokens"]), float(marginal_total["effective_tokens"]), ), - "usd_percent": _saving( - float(baseline_total["effective_usd"]), - float(marginal_total["effective_usd"]), + "usd_percent": _optional_saving( + baseline_total["effective_usd"], marginal_total["effective_usd"] ), "latency_percent": _saving( float(baseline_total["effective_latency_ms"]), @@ -290,18 +303,20 @@ def efficiency(total: dict[str, Any], *, include_governance: bool) -> dict[str, "tool_calls_percent": gross_savings["tool_calls_percent"], "repeated_calls_percent": gross_savings["repeated_calls_percent"], "confidence_level": confidence_level, - "tokens_confidence_interval": list(net_token_ci), - "tokens_95pct_ci": list(net_token_ci), + "tokens_confidence_interval": list(net_token_ci) if quality_evaluable else None, + "tokens_95pct_ci": list(net_token_ci) if quality_evaluable else None, } quality_margin_value = float(quality_margin_pp) - quality_preserved = delta_pp >= -quality_margin_value + quality_preserved = delta_pp >= -quality_margin_value if quality_evaluable else None has_verified_success = int(marginal_total["resolved"]) > 0 false_stop_rate = marginal_total["false_stop_rate"] false_stops_acceptable = false_stop_rate is None or float(false_stop_rate) <= float( max_false_stop_rate ) - if not quality_preserved: + if not quality_evaluable: + intervention_status = "pass_through" + elif not quality_preserved: intervention_status = "quality_regression" elif not false_stops_acceptable: intervention_status = "false_stop_risk" @@ -339,7 +354,8 @@ def efficiency(total: dict[str, Any], *, include_governance: bool) -> dict[str, "resolved_delta_pp": delta_pp, "non_inferiority_margin_pp": quality_margin_pp, "preserved_within_margin": quality_preserved, - "preserved_within_one_pp": delta_pp >= -1.0, + "preserved_within_one_pp": delta_pp >= -1.0 if quality_evaluable else None, + "evaluable": quality_evaluable, "regressions": sum( baseline[item].resolved and not marginal[item].resolved for item in ids ), @@ -359,7 +375,7 @@ def efficiency(total: dict[str, Any], *, include_governance: bool) -> dict[str, "net_positive": float(cast(float, net_savings["tokens_percent"])) > 0.0, "graceful_irrelevance": intervention_status == "pass_through", "eligible_for_support": ( - quality_preserved and false_stops_acceptable and has_verified_success + quality_preserved is True and false_stops_acceptable and has_verified_success ), }, } @@ -391,6 +407,10 @@ def render_public_report(result: dict[str, Any]) -> str: margin = float(quality["non_inferiority_margin_pp"]) intervention = result.get("intervention", {"status": "unclassified"}) governance = result.get("governance", {"tokens": 0, "usd": 0.0, "latency_ms": 0}) + usd_change = "n/a" if savings["usd_percent"] is None else f"{savings['usd_percent']:.2f}% lower" + quality_result = ( + quality["preserved_within_margin"] if quality.get("evaluable", True) else "not evaluable" + ) return "\n".join( [ "# Measured public benchmark comparison", @@ -414,8 +434,10 @@ def render_public_report(result: dict[str, Any]) -> str: f"{marginal['effective_tokens']:,} | {savings['tokens_percent']:.2f}% fewer |" ), ( - f"| Effective USD | ${baseline['effective_usd']:.4f} | " - f"${marginal['effective_usd']:.4f} | {savings['usd_percent']:.2f}% lower |" + "| Effective USD | " + f"{_format_optional_usd(baseline['effective_usd'])} | " + f"{_format_optional_usd(marginal['effective_usd'])} | " + f"{usd_change} |" ), ( f"| Effective latency | {baseline['effective_latency_ms']:,} ms | " @@ -446,7 +468,8 @@ def render_public_report(result: dict[str, Any]) -> str: "", ( f"MARGINAL overhead: **{governance['tokens']:,} tokens**, " - f"**${governance['usd']:.6f}**, **{governance['latency_ms']:,} ms**." + f"**{_format_optional_usd(governance['usd'])}**, " + f"**{governance['latency_ms']:,} ms**." ), ( f"Gross agent-token savings: **{gross['tokens_percent']:.2f}%**. " @@ -458,10 +481,12 @@ def render_public_report(result: dict[str, Any]) -> str: ( f"Net token savings {confidence_percent:.1f}% bootstrap interval: " f"**{ci[0]:.2f}% to {ci[1]:.2f}%**." + if quality.get("evaluable", True) + else "Token uncertainty: **not evaluable** without a successful task in both arms." ), ( f"Quality preserved within the {margin:.2f} pp non-inferiority margin: " - f"**{quality['preserved_within_margin']}**." + f"**{quality_result}**." ), f"Regressions: **{quality['regressions']}**. Recoveries: **{quality['recoveries']}**.", ( diff --git a/src/marginal/receipts.py b/src/marginal/receipts.py index fd83670..50e6fb2 100644 --- a/src/marginal/receipts.py +++ b/src/marginal/receipts.py @@ -12,6 +12,7 @@ from typing import Any from .canonical import canonical_hash +from .reason_codes import REASON_CODE_VERSION, ReasonCode RECEIPT_SCHEMA_VERSION = "1.0" PROGRESS_EVIDENCE_SCHEMA_VERSION = "1.0" @@ -249,6 +250,8 @@ def __post_init__(self) -> None: "enforcement_level", ): _required_text(getattr(self, name), name) + if self.reason_code not in {code.value for code in ReasonCode}: + raise ValueError(f"unsupported reason_code for registry version {REASON_CODE_VERSION}") if not isinstance(self.decision_hash, str): raise TypeError("decision_hash must be a string") for name in ("state_hash", "evidence_hash", "trajectory_hash"): diff --git a/tests/test_public_eval_v2.py b/tests/test_public_eval_v2.py index 70c7e40..c31b54a 100644 --- a/tests/test_public_eval_v2.py +++ b/tests/test_public_eval_v2.py @@ -106,3 +106,22 @@ def test_public_report_includes_efficiency_and_configured_criterion() -> None: assert "USD per resolved task" in report assert "90.0% bootstrap interval" in report assert "0.50 pp non-inferiority margin" in report + + +def test_unmeasured_usd_and_zero_success_quality_are_not_claimed() -> None: + from marginal.public_eval import RunRecord, compare_runs, render_public_report + + baseline = { + "task": RunRecord(instance_id="task", resolved=False, tokens=100, usd_measured=False) + } + marginal = { + "task": RunRecord(instance_id="task", resolved=False, tokens=50, usd_measured=False) + } + result = compare_runs(baseline, marginal, bootstrap_samples=20) + report = render_public_report(result) + + assert result["baseline"]["effective_usd"] is None + assert result["quality"]["preserved_within_margin"] is None + assert result["net_savings"]["tokens_confidence_interval"] is None + assert "not evaluable" in report + assert "bootstrap interval" not in report diff --git a/tests/test_receipts.py b/tests/test_receipts.py index eaed20b..41f03f2 100644 --- a/tests/test_receipts.py +++ b/tests/test_receipts.py @@ -7,6 +7,7 @@ import pytest from jsonschema import Draft202012Validator +from marginal.reason_codes import ReasonCode from marginal.receipts import ( DecisionReceipt, GovernanceCost, @@ -64,6 +65,16 @@ def test_receipt_hash_binds_the_canonical_payload_and_detects_tampering() -> Non assert not verify_decision_receipt(replace(receipt, decision="allow")) +@pytest.mark.parametrize("reason_code", [code.value for code in ReasonCode]) +def test_decision_receipt_accepts_only_versioned_reason_codes(reason_code: str) -> None: + assert replace(_receipt(), reason_code=reason_code).reason_code == reason_code + + +def test_decision_receipt_rejects_unknown_reason_code() -> None: + with pytest.raises(ValueError, match="unsupported reason_code"): + replace(_receipt(), reason_code="this_is_not_a_real_reason_code") + + def test_receipt_payload_keeps_unavailable_measurements_explicit_and_mappings_immutable() -> None: """Catches omitting unavailable evidence or retaining a caller-mutable attestation map."""