From 5c24444678b6a054915bf3a8bf46694e2a2955c3 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Mon, 17 Aug 2026 20:06:57 +0000 Subject: [PATCH] experiment: seal critical PTCNA evaluation result --- docs/architecture.md | 7 +++ docs/evaluations/ptcna-critical-result-v1.md | 52 ++++++++++++++++++++ ptcna/data/ptcna-critical-result-v1.json | 28 +++++++++++ ptcna/tests/test_critical_evaluation.py | 36 +++++++++++--- ptcna_msdmd.ts | 4 +- 5 files changed, 119 insertions(+), 8 deletions(-) create mode 100644 docs/evaluations/ptcna-critical-result-v1.md create mode 100644 ptcna/data/ptcna-critical-result-v1.json diff --git a/docs/architecture.md b/docs/architecture.md index 6a9dbf1..f261804 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -124,6 +124,13 @@ METAPAT canon. ## Status log +- **2026-08-17 — critical role-acquisition evaluation FALSIFIED.** The frozen + 18-case, five-repetition program produced target accuracy `0.3333333333` + against fallback accuracy `0.9444444444`. The target failed its absolute + `0.75` usefulness threshold and its `0.05` superiority-margin threshold. + Construction compatibility remains established; usefulness and superiority + are falsified only for this declared in-sample scope. + - **2026-08-17 — executable construction/evaluation boundary.** Added the four-layer target receipt, independently test-backed hashed-linear fallback, explicit attributed failover, and frozen evaluation/verdict API. Removed the diff --git a/docs/evaluations/ptcna-critical-result-v1.md b/docs/evaluations/ptcna-critical-result-v1.md new file mode 100644 index 0000000..df7e57f --- /dev/null +++ b/docs/evaluations/ptcna-critical-result-v1.md @@ -0,0 +1,52 @@ +# PTCNA critical role-acquisition evaluation — result v1 + +Status recorded before repair: + +- PTCNA meets its frozen usefulness threshold: `FALSIFIED`. +- PTCNA outperforms the simpler hashed-linear fallback: `FALSIFIED`. + +The preregistered plan was merged at +`f2028004cbee65f4a0a8c113e283bd38b29d1e2c` before execution. Its semantic +digest is `67cdad3aefb3e33f6fbf3994de54e1b73a01105527bf241fce08947ed7046bbe`. +The sealed machine result is +`ptcna/data/ptcna-critical-result-v1.json`, digest +`3d73f08f3e0eeaf0f5e508aba762844c9e8d11b7146b654fd562aff425e30f13`. + +## Frozen execution + +- Workload: 18 balanced cases over the declared `phi` cognitive, `psi` + self-model, and `omega` autonomy roles. +- Target: `ptcna.experimental.v1`. +- Comparator: `fallback.hashed-linear.v1`. +- Training: three epochs, reward `1.0`. +- Repetitions: five fresh target/comparator constructions. +- Aggregation: micro mean over 90 post-training case evaluations. +- Usefulness threshold: target accuracy at least `0.75`. +- Superiority threshold: target advantage over fallback at least `0.05`. +- Resource use: 270/270 training steps, 90/90 evaluations, 46.675 seconds + under the frozen 120-second limit. +- Failure: none. + +## Result + +| measure | result | +|---|---:| +| PTCNA target accuracy | 0.3333333333333333 | +| hashed-linear fallback accuracy | 0.9444444444444444 | +| target advantage | -0.6111111111111112 | + +An implementation-independent arithmetic replay reads only the sealed JSON and +reapplies the two frozen inequalities. It agrees with both recorded verdicts. + +## Standing + +This falsifies usefulness for the tested in-sample role-acquisition scope and +falsifies superiority to this matched task-interface fallback. It does not +revoke the already established construction compatibility or replace the +selected architecture with the comparator. It establishes no generalization, +continuous seven-fold geometry, EDCM measurement validity, external validity, +or production privacy result. + +No workload, architecture, reward, epoch count, metric, threshold, repetition, +resource rule, backend identity, or failure rule was changed after outcome +inspection. diff --git a/ptcna/data/ptcna-critical-result-v1.json b/ptcna/data/ptcna-critical-result-v1.json new file mode 100644 index 0000000..f39c74e --- /dev/null +++ b/ptcna/data/ptcna-critical-result-v1.json @@ -0,0 +1,28 @@ +{ + "claim_rules": { + "parity": "FALSIFIED for superiority only", + "scope": "in-sample acquisition on the declared phi/psi/omega role surface; no generalization, geometry, EDCM, external-validity, or privacy claim", + "superiority": "target_accuracy - comparator_accuracy >= minimum_target_advantage_vs_fallback", + "usefulness": "target_accuracy >= minimum_target_accuracy" + }, + "plan_digest": "67cdad3aefb3e33f6fbf3994de54e1b73a01105527bf241fce08947ed7046bbe", + "receipt": { + "case_evaluations": 90, + "comparator_accuracy": 0.9444444444444444, + "comparator_backend": "fallback.hashed-linear.v1", + "duration_ms": 46675.045, + "failure_reason": null, + "plan_digest": "67cdad3aefb3e33f6fbf3994de54e1b73a01105527bf241fce08947ed7046bbe", + "plan_id": "ptcna-critical-role-acquisition-v1", + "superiority_status": "FALSIFIED", + "target_accuracy": 0.3333333333333333, + "target_advantage_vs_fallback": -0.6111111111111112, + "target_backend": "ptcna.experimental.v1", + "training_steps": 270, + "usefulness_status": "FALSIFIED" + }, + "result_digest": "3d73f08f3e0eeaf0f5e508aba762844c9e8d11b7146b654fd562aff425e30f13", + "schema": "ptcna.critical-evaluation-result", + "schema_version": "1.0.0", + "source_commit": "3d67f359af05e58c1e426dad14621d0e69b5ec4c" +} diff --git a/ptcna/tests/test_critical_evaluation.py b/ptcna/tests/test_critical_evaluation.py index 235ab65..44f0eeb 100644 --- a/ptcna/tests/test_critical_evaluation.py +++ b/ptcna/tests/test_critical_evaluation.py @@ -1,6 +1,10 @@ """Checks for the frozen critical-evaluation plan and receipt sealing.""" +import json +from pathlib import Path + from ptcna.critical_evaluation import _canonical_digest, load_frozen_plan +from ptcna.evaluation import FALSIFIED, SURVIVED_NOT_PROVED # === CHECKS === # id: check_ptcna_critical_plan_digest @@ -13,7 +17,7 @@ # # id: check_ptcna_critical_result_digest # proves: ptcna_critical_result_content_addressed -# call: self::test_result_digest_is_content_sensitive +# call: self::test_sealed_result_digest_and_independent_verdict_replay # requires: python3 # timeout: 30 # mutates: none @@ -34,8 +38,28 @@ def test_critical_plan_is_balanced_and_digest_locked() -> None: assert len(_canonical_digest(artifact)) == 64 -def test_result_digest_is_content_sensitive() -> None: - first = {"plan_digest": "a" * 64, "usefulness_status": "FALSIFIED"} - second = {"plan_digest": "a" * 64, "usefulness_status": "UNRESOLVED"} - assert _canonical_digest(first) == _canonical_digest(dict(first)) - assert _canonical_digest(first) != _canonical_digest(second) +def test_sealed_result_digest_and_independent_verdict_replay() -> None: + result_path = ( + Path(__file__).resolve().parents[1] + / "data/ptcna-critical-result-v1.json" + ) + result = json.loads(result_path.read_text(encoding="utf-8")) + recorded_digest = result.pop("result_digest") + assert _canonical_digest(result) == recorded_digest + + plan, _ = load_frozen_plan() + receipt = result["receipt"] + target = receipt["target_accuracy"] + comparator = receipt["comparator_accuracy"] + usefulness = ( + SURVIVED_NOT_PROVED + if target >= plan.minimum_target_accuracy + else FALSIFIED + ) + superiority = ( + SURVIVED_NOT_PROVED + if target - comparator >= plan.minimum_target_advantage_vs_fallback + else FALSIFIED + ) + assert usefulness == receipt["usefulness_status"] == FALSIFIED + assert superiority == receipt["superiority_status"] == FALSIFIED diff --git a/ptcna_msdmd.ts b/ptcna_msdmd.ts index d7ab14f..7c33a1b 100644 --- a/ptcna_msdmd.ts +++ b/ptcna_msdmd.ts @@ -1444,7 +1444,7 @@ export default defineMsdmdCollection({ { "block": "CHECKS", "fields": { - "call": "self::test_result_digest_is_content_sensitive", + "call": "self::test_sealed_result_digest_and_independent_verdict_replay", "cleanup": "none", "mutates": "none", "proves": "ptcna_critical_result_content_addressed", @@ -2342,7 +2342,7 @@ export default defineMsdmdCollection({ "kind": "calls", "source_block": "CHECKS", "source_id": "check_ptcna_critical_result_digest", - "to": "self::test_result_digest_is_content_sensitive" + "to": "self::test_sealed_result_digest_and_independent_verdict_replay" }, { "from": "check_ptcna_critical_result_digest",