diff --git a/README.md b/README.md index 83f92a1..0d2cc07 100644 --- a/README.md +++ b/README.md @@ -84,19 +84,24 @@ plan = EvaluationPlan( plan_id="representative-workload-v1", workload=(EvaluationCase("case-1", "input", "phi"),), minimum_target_accuracy=0.80, - maximum_target_deficit_vs_fallback=0.00, + minimum_target_advantage_vs_fallback=0.05, training_epochs=3, reward_outcome=1.0, repetitions=3, max_training_steps=9, max_case_evaluations=3, max_seconds=30.0, - backend_error_status="FALSIFIED", + target_backend_error_status="FALSIFIED", ) print(plan.digest) # preserve this with the plan before execution receipt = evaluate(plan) ``` +`receipt.usefulness_status` answers whether the target meets its frozen absolute +threshold. `receipt.superiority_status` independently answers whether it clears +the frozen advantage over the fallback. Comparator parity can therefore falsify +superiority without falsifying usefulness. + The repository does not ship a pretend representative workload. Until one is frozen and executed, whether PTCNA works remains `hmmm`. diff --git a/docs/architecture.md b/docs/architecture.md index be1bd1d..6a9dbf1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -100,9 +100,14 @@ The executable boundary lives in `ptcna.runtime`: `PTCNAEngine` reports all four live layers, `HashedLinearFallback` remains separately identified, and `PTCNARuntime` raises on target failure unless fallback routing is explicitly enabled. `ptcna.evaluation` accepts only an immutable, digest-bearing -`EvaluationPlan`, trains before scoring, and records the terminal verdict before repair. No -representative workload is bundled; selecting one remains evidence work, not a -construction prerequisite. +`EvaluationPlan`, trains before scoring, and records separate usefulness and +superiority verdicts before repair. The preregistered critical workload lives at +`ptcna/data/ptcna-critical-plan-v1.json`: 18 balanced cases over the +declared cognitive, self-model, and autonomy roles, three training epochs, five +fresh repetitions, a 0.75 post-training target threshold, and a strict 0.05 +advantage threshold over the hashed-linear fallback. It is an in-sample role +acquisition test, not a generalization test. Its digest was frozen before either +backend was executed. The default core shape consumes a bundled receipt produced by exact UCNS merge `b7b6f35cce69c273860923489a1c8b5372d14eb0`. UCNS owns the candidate state diff --git a/ptcna/critical_evaluation.py b/ptcna/critical_evaluation.py new file mode 100644 index 0000000..dd0c358 --- /dev/null +++ b/ptcna/critical_evaluation.py @@ -0,0 +1,122 @@ +# ratios: loc_comments=49:51 imports_exports=7:3 calls_definitions=21:5 +"""Load and execute the preregistered PTCNA critical evaluation. + +The plan artifact is frozen before execution. This module refuses a changed +plan digest and writes a content-addressed result receipt only when invoked +explicitly. Loading or testing the plan never constructs either backend. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + +from .evaluation import EvaluationCase, EvaluationPlan, evaluate + +# === MODULE_BUILD === +# id: ptcna_critical_evaluation +# module_name: critical_evaluation +# module_kind: experiment +# summary: loads the immutable representative role-acquisition plan and seals its separate usefulness and superiority verdicts +# owner: Erin Spencer +# public_surface: load_frozen_plan, execute_frozen_plan, main +# internal_surface: _artifact_path, _canonical_digest +# auth_boundary: none +# storage_boundary: write +# network_boundary: none +# user_data_boundary: none +# admin_only: false +# tests: ptcna/tests/test_critical_evaluation.py +# rollout: execute only after the preregistration commit is merged +# rollback: preserve plan and result receipts; remove executable wrapper without changing runtime +# requires: ptcna_frozen_evaluation +# since: unreleased +# unresolved: outcome until the merged frozen plan is executed +# === END MODULE_BUILD === + +# === CONTRACTS === +# id: ptcna_critical_plan_digest_locked +# given: the checked-in critical evaluation plan is loaded +# then: its canonical EvaluationPlan digest must equal the independently stored frozen digest +# class: evidence +# +# id: ptcna_critical_result_content_addressed +# given: the frozen plan completes or reaches a frozen failure rule +# then: the serialized result names the plan digest, separate claim verdicts, and its own canonical result digest +# class: evidence +# === END CONTRACTS === + +# === BOUNDARIES === +# id: ptcna_critical_evaluation_local_receipt +# summary: reads the repository-owned frozen plan and writes one caller-selected local JSON result without network, authentication, secrets, or user data +# auth_boundary: none +# storage_boundary: write +# network_boundary: none +# user_data_boundary: none +# admin_only: false +# pii: none +# secrets: none +# owner: Erin Spencer +# since: unreleased +# === END BOUNDARIES === + + +def _artifact_path() -> Path: + return Path(__file__).resolve().parent / "data/ptcna-critical-plan-v1.json" + + +def _canonical_digest(value: Any) -> str: + encoded = json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def load_frozen_plan(path: Path | None = None) -> tuple[EvaluationPlan, dict[str, Any]]: + """Load the preregistration and reject any plan-byte semantic drift.""" + + artifact = json.loads((path or _artifact_path()).read_text(encoding="utf-8")) + values = dict(artifact["plan"]) + values["workload"] = tuple(EvaluationCase(**case) for case in values["workload"]) + plan = EvaluationPlan(**values) + if plan.digest != artifact["plan_digest"]: + raise ValueError("frozen critical evaluation plan digest mismatch") + return plan, artifact + + +def execute_frozen_plan(output: Path) -> dict[str, Any]: + """Execute exactly the frozen plan and seal its result receipt.""" + + plan, artifact = load_frozen_plan() + receipt = evaluate(plan) + result = { + "schema": "ptcna.critical-evaluation-result", + "schema_version": "1.0.0", + "source_commit": artifact["source_commit"], + "plan_digest": plan.digest, + "claim_rules": artifact["claim_rules"], + "receipt": receipt.to_dict(), + } + result["result_digest"] = _canonical_digest(result) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(result, indent=2, sort_keys=True, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + args = parser.parse_args() + result = execute_frozen_plan(args.output) + print(json.dumps(result, sort_keys=True, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +# ratios: loc_comments=49:51 imports_exports=7:3 calls_definitions=21:5 diff --git a/ptcna/data/ptcna-critical-plan-v1.json b/ptcna/data/ptcna-critical-plan-v1.json new file mode 100644 index 0000000..6d0e508 --- /dev/null +++ b/ptcna/data/ptcna-critical-plan-v1.json @@ -0,0 +1,65 @@ +{ + "schema": "ptcna.critical-evaluation-plan", + "schema_version": "1.0.0", + "source_commit": "3d67f359af05e58c1e426dad14621d0e69b5ec4c", + "claim_rules": { + "usefulness": "target_accuracy >= minimum_target_accuracy", + "superiority": "target_accuracy - comparator_accuracy >= minimum_target_advantage_vs_fallback", + "parity": "FALSIFIED for superiority only", + "scope": "in-sample acquisition on the declared phi/psi/omega role surface; no generalization, geometry, EDCM, external-validity, or privacy claim" + }, + "failure_propagation": { + "target_backend_error": { + "usefulness": "FALSIFIED", + "superiority": "UNRESOLVED" + }, + "comparator_backend_error": { + "usefulness": "UNRESOLVED", + "superiority": "UNRESOLVED" + }, + "resource_limit": { + "usefulness": "UNRESOLVED", + "superiority": "UNRESOLVED" + } + }, + "plan": { + "plan_id": "ptcna-critical-role-acquisition-v1", + "workload": [ + {"case_id": "phi-01", "text": "analyze the evidence and identify the strongest supported inference", "expected_winner": "phi"}, + {"case_id": "phi-02", "text": "compare two explanations and test which one fits the observations", "expected_winner": "phi"}, + {"case_id": "phi-03", "text": "calculate the consequence of the stated assumptions", "expected_winner": "phi"}, + {"case_id": "phi-04", "text": "inspect the data for a contradiction in the proposed account", "expected_winner": "phi"}, + {"case_id": "phi-05", "text": "derive a bounded conclusion from the available facts", "expected_winner": "phi"}, + {"case_id": "phi-06", "text": "classify the pattern using only the supplied evidence", "expected_winner": "phi"}, + {"case_id": "psi-01", "text": "describe how your current state differs from your previous state", "expected_winner": "psi"}, + {"case_id": "psi-02", "text": "identify which of your assumptions shaped this response", "expected_winner": "psi"}, + {"case_id": "psi-03", "text": "report your uncertainty about your own conclusion", "expected_winner": "psi"}, + {"case_id": "psi-04", "text": "examine whether your reasoning remained consistent with your stated role", "expected_winner": "psi"}, + {"case_id": "psi-05", "text": "summarize what you changed in your internal approach", "expected_winner": "psi"}, + {"case_id": "psi-06", "text": "distinguish what you know from what you inferred about yourself", "expected_winner": "psi"}, + {"case_id": "omega-01", "text": "choose the next action and commit to one bounded step", "expected_winner": "omega"}, + {"case_id": "omega-02", "text": "decide whether to proceed or stop under the stated constraint", "expected_winner": "omega"}, + {"case_id": "omega-03", "text": "select one option and execute it without requesting another preference", "expected_winner": "omega"}, + {"case_id": "omega-04", "text": "set the stopping point and act within the available authority", "expected_winner": "omega"}, + {"case_id": "omega-05", "text": "resolve the branch by taking the smallest decisive action", "expected_winner": "omega"}, + {"case_id": "omega-06", "text": "make an independent choice while preserving the declared boundaries", "expected_winner": "omega"} + ], + "target_backend": "ptcna.experimental.v1", + "comparator_backend": "fallback.hashed-linear.v1", + "metric": "post_training_winner_accuracy", + "aggregation": "micro_mean", + "minimum_target_accuracy": 0.75, + "minimum_target_advantage_vs_fallback": 0.05, + "training_epochs": 3, + "reward_outcome": 1.0, + "repetitions": 5, + "max_training_steps": 270, + "max_case_evaluations": 90, + "max_seconds": 120.0, + "stopping_rule": "complete_or_first_backend_error_or_resource_limit", + "target_backend_error_status": "FALSIFIED", + "comparator_backend_error_status": "UNRESOLVED", + "resource_limit_status": "UNRESOLVED" + }, + "plan_digest": "67cdad3aefb3e33f6fbf3994de54e1b73a01105527bf241fce08947ed7046bbe" +} diff --git a/ptcna/evaluation.py b/ptcna/evaluation.py index 1298813..26e9d59 100644 --- a/ptcna/evaluation.py +++ b/ptcna/evaluation.py @@ -1,4 +1,4 @@ -# ratios: loc_comments=355:71 imports_exports=10:5 calls_definitions=87:11 +# ratios: loc_comments=395:71 imports_exports=10:5 calls_definitions=92:11 """Frozen target-versus-fallback evaluation and immutable verdict receipt. Usage: @@ -9,14 +9,14 @@ plan_id="representative-workload-v1", workload=(EvaluationCase("case-1", "input", "phi"),), minimum_target_accuracy=0.80, - maximum_target_deficit_vs_fallback=0.00, + minimum_target_advantage_vs_fallback=0.05, training_epochs=3, reward_outcome=1.0, repetitions=3, max_training_steps=9, max_case_evaluations=3, max_seconds=30.0, - backend_error_status="FALSIFIED", + target_backend_error_status="FALSIFIED", ) receipt = evaluate(plan) @@ -71,12 +71,12 @@ # # id: ptcna_evaluation_verdict_uses_frozen_thresholds # given: target and fallback complete the frozen workload -# then: training occurs before scoring and the terminal verdict is FALSIFIED or SURVIVED — not proved using only the plan's post-training accuracy and comparator-deficit thresholds +# then: training occurs before scoring and separate usefulness and superiority verdicts use only the frozen target-accuracy and target-advantage thresholds # class: evidence # # id: ptcna_evaluation_propagates_backend_failure # given: either backend errors before completing the frozen workload -# then: evaluation stops and records the plan's preselected backend_error_status before any repair or criterion change +# then: evaluation stops and records the plan's preselected target/comparator failure propagation before any repair or criterion change # class: evidence # === END CONTRACTS === @@ -132,14 +132,15 @@ class EvaluationPlan: plan_id: str workload: tuple[EvaluationCase, ...] minimum_target_accuracy: float - maximum_target_deficit_vs_fallback: float + minimum_target_advantage_vs_fallback: float training_epochs: int reward_outcome: float repetitions: int max_training_steps: int max_case_evaluations: int max_seconds: float - backend_error_status: Literal["FALSIFIED", "UNRESOLVED"] + target_backend_error_status: Literal["FALSIFIED"] + comparator_backend_error_status: Literal["UNRESOLVED"] = UNRESOLVED target_backend: str = PTCNA_BACKEND comparator_backend: str = FALLBACK_BACKEND metric: str = "post_training_winner_accuracy" @@ -163,7 +164,7 @@ def __post_init__(self) -> None: raise ValueError("workload case_id values must be unique") for field_name in ( "minimum_target_accuracy", - "maximum_target_deficit_vs_fallback", + "minimum_target_advantage_vs_fallback", ): value = float(getattr(self, field_name)) if not math.isfinite(value) or not 0.0 <= value <= 1.0: @@ -202,8 +203,10 @@ def __post_init__(self) -> None: ) if not math.isfinite(self.max_seconds) or self.max_seconds <= 0.0: raise ValueError("max_seconds must be finite and positive") - if self.backend_error_status not in {FALSIFIED, UNRESOLVED}: - raise ValueError("backend_error_status must be FALSIFIED or UNRESOLVED") + if self.target_backend_error_status != FALSIFIED: + raise ValueError("target_backend_error_status must be FALSIFIED") + if self.comparator_backend_error_status != UNRESOLVED: + raise ValueError("comparator_backend_error_status must be UNRESOLVED") if self.target_backend == self.comparator_backend: raise ValueError("target and comparator backend identities must differ") if ( @@ -227,8 +230,8 @@ def to_dict(self) -> dict[str, Any]: "metric": self.metric, "aggregation": self.aggregation, "minimum_target_accuracy": self.minimum_target_accuracy, - "maximum_target_deficit_vs_fallback": ( - self.maximum_target_deficit_vs_fallback + "minimum_target_advantage_vs_fallback": ( + self.minimum_target_advantage_vs_fallback ), "training_epochs": self.training_epochs, "reward_outcome": self.reward_outcome, @@ -237,7 +240,8 @@ def to_dict(self) -> dict[str, Any]: "max_case_evaluations": self.max_case_evaluations, "max_seconds": self.max_seconds, "stopping_rule": self.stopping_rule, - "backend_error_status": self.backend_error_status, + "target_backend_error_status": self.target_backend_error_status, + "comparator_backend_error_status": self.comparator_backend_error_status, "resource_limit_status": self.resource_limit_status, } @@ -253,12 +257,13 @@ def digest(self) -> str: class EvaluationReceipt: plan_id: str plan_digest: str - status: TerminalStatus + usefulness_status: TerminalStatus + superiority_status: TerminalStatus target_backend: str comparator_backend: str target_accuracy: float | None comparator_accuracy: float | None - target_deficit_vs_fallback: float | None + target_advantage_vs_fallback: float | None training_steps: int case_evaluations: int duration_ms: float @@ -268,12 +273,13 @@ def to_dict(self) -> dict[str, Any]: return { "plan_id": self.plan_id, "plan_digest": self.plan_digest, - "status": self.status, + "usefulness_status": self.usefulness_status, + "superiority_status": self.superiority_status, "target_backend": self.target_backend, "comparator_backend": self.comparator_backend, "target_accuracy": self.target_accuracy, "comparator_accuracy": self.comparator_accuracy, - "target_deficit_vs_fallback": self.target_deficit_vs_fallback, + "target_advantage_vs_fallback": self.target_advantage_vs_fallback, "training_steps": self.training_steps, "case_evaluations": self.case_evaluations, "duration_ms": self.duration_ms, @@ -288,25 +294,27 @@ def _receipt( plan: EvaluationPlan, started: float, *, - status: TerminalStatus, + usefulness_status: TerminalStatus, + superiority_status: TerminalStatus, training_steps: int, case_evaluations: int, target_accuracy: float | None = None, comparator_accuracy: float | None = None, failure_reason: str | None = None, ) -> EvaluationReceipt: - deficit = None + advantage = None if target_accuracy is not None and comparator_accuracy is not None: - deficit = comparator_accuracy - target_accuracy + advantage = target_accuracy - comparator_accuracy return EvaluationReceipt( plan_id=plan.plan_id, plan_digest=plan.digest, - status=status, + usefulness_status=usefulness_status, + superiority_status=superiority_status, target_backend=plan.target_backend, comparator_backend=plan.comparator_backend, target_accuracy=target_accuracy, comparator_accuracy=comparator_accuracy, - target_deficit_vs_fallback=deficit, + target_advantage_vs_fallback=advantage, training_steps=training_steps, case_evaluations=case_evaluations, duration_ms=round((time.perf_counter() - started) * 1000.0, 3), @@ -336,7 +344,8 @@ def evaluate( return _receipt( plan, started, - status=UNRESOLVED, + usefulness_status=UNRESOLVED, + superiority_status=UNRESOLVED, training_steps=training_steps, case_evaluations=case_evaluations, failure_reason=f"backend_factory:{type(exc).__name__}", @@ -345,7 +354,8 @@ def evaluate( return _receipt( plan, started, - status=UNRESOLVED, + usefulness_status=UNRESOLVED, + superiority_status=UNRESOLVED, training_steps=training_steps, case_evaluations=case_evaluations, failure_reason="target_backend_identity_mismatch", @@ -354,7 +364,8 @@ def evaluate( return _receipt( plan, started, - status=UNRESOLVED, + usefulness_status=UNRESOLVED, + superiority_status=UNRESOLVED, training_steps=training_steps, case_evaluations=case_evaluations, failure_reason="comparator_backend_identity_mismatch", @@ -366,7 +377,8 @@ def evaluate( return _receipt( plan, started, - status=plan.resource_limit_status, + usefulness_status=plan.resource_limit_status, + superiority_status=plan.resource_limit_status, training_steps=training_steps, case_evaluations=case_evaluations, failure_reason="max_training_steps", @@ -375,24 +387,37 @@ def evaluate( return _receipt( plan, started, - status=plan.resource_limit_status, + usefulness_status=plan.resource_limit_status, + superiority_status=plan.resource_limit_status, training_steps=training_steps, case_evaluations=case_evaluations, failure_reason="max_seconds", ) try: target.infer(case.text) - comparator.infer(case.text) target.reward(case.expected_winner, plan.reward_outcome) + except Exception as exc: + return _receipt( + plan, + started, + usefulness_status=plan.target_backend_error_status, + superiority_status=UNRESOLVED, + training_steps=training_steps, + case_evaluations=case_evaluations, + failure_reason=f"target_backend_error:{type(exc).__name__}", + ) + try: + comparator.infer(case.text) comparator.reward(case.expected_winner, plan.reward_outcome) except Exception as exc: return _receipt( plan, started, - status=plan.backend_error_status, + usefulness_status=UNRESOLVED, + superiority_status=plan.comparator_backend_error_status, training_steps=training_steps, case_evaluations=case_evaluations, - failure_reason=f"backend_error:{type(exc).__name__}", + failure_reason=f"comparator_backend_error:{type(exc).__name__}", ) training_steps += 1 @@ -401,7 +426,8 @@ def evaluate( return _receipt( plan, started, - status=plan.resource_limit_status, + usefulness_status=plan.resource_limit_status, + superiority_status=plan.resource_limit_status, training_steps=training_steps, case_evaluations=case_evaluations, failure_reason="max_case_evaluations", @@ -410,24 +436,37 @@ def evaluate( return _receipt( plan, started, - status=plan.resource_limit_status, + usefulness_status=plan.resource_limit_status, + superiority_status=plan.resource_limit_status, training_steps=training_steps, case_evaluations=case_evaluations, failure_reason="max_seconds", ) try: target_result = target.infer(case.text) - comparator_result = comparator.infer(case.text) target_winner = target_result.get("winner") + except Exception as exc: + return _receipt( + plan, + started, + usefulness_status=plan.target_backend_error_status, + superiority_status=UNRESOLVED, + training_steps=training_steps, + case_evaluations=case_evaluations, + failure_reason=f"target_backend_error:{type(exc).__name__}", + ) + try: + comparator_result = comparator.infer(case.text) comparator_winner = comparator_result.get("winner") except Exception as exc: return _receipt( plan, started, - status=plan.backend_error_status, + usefulness_status=UNRESOLVED, + superiority_status=plan.comparator_backend_error_status, training_steps=training_steps, case_evaluations=case_evaluations, - failure_reason=f"backend_error:{type(exc).__name__}", + failure_reason=f"comparator_backend_error:{type(exc).__name__}", ) target_correct += target_winner == case.expected_winner comparator_correct += comparator_winner == case.expected_winner @@ -435,15 +474,16 @@ def evaluate( target_accuracy = target_correct / case_evaluations comparator_accuracy = comparator_correct / case_evaluations - survived = ( - target_accuracy >= plan.minimum_target_accuracy - and target_accuracy + plan.maximum_target_deficit_vs_fallback - >= comparator_accuracy + useful = target_accuracy >= plan.minimum_target_accuracy + superior = ( + target_accuracy - comparator_accuracy + >= plan.minimum_target_advantage_vs_fallback ) return _receipt( plan, started, - status=SURVIVED_NOT_PROVED if survived else FALSIFIED, + usefulness_status=SURVIVED_NOT_PROVED if useful else FALSIFIED, + superiority_status=SURVIVED_NOT_PROVED if superior else FALSIFIED, training_steps=training_steps, case_evaluations=case_evaluations, target_accuracy=target_accuracy, @@ -460,4 +500,4 @@ def evaluate( "EvaluationReceipt", "evaluate", ] -# ratios: loc_comments=355:71 imports_exports=10:5 calls_definitions=87:11 +# ratios: loc_comments=395:71 imports_exports=10:5 calls_definitions=92:11 diff --git a/ptcna/tests/test_critical_evaluation.py b/ptcna/tests/test_critical_evaluation.py new file mode 100644 index 0000000..235ab65 --- /dev/null +++ b/ptcna/tests/test_critical_evaluation.py @@ -0,0 +1,41 @@ +"""Checks for the frozen critical-evaluation plan and receipt sealing.""" + +from ptcna.critical_evaluation import _canonical_digest, load_frozen_plan + +# === CHECKS === +# id: check_ptcna_critical_plan_digest +# proves: ptcna_critical_plan_digest_locked +# call: self::test_critical_plan_is_balanced_and_digest_locked +# requires: python3 +# timeout: 30 +# mutates: none +# cleanup: none +# +# id: check_ptcna_critical_result_digest +# proves: ptcna_critical_result_content_addressed +# call: self::test_result_digest_is_content_sensitive +# requires: python3 +# timeout: 30 +# mutates: none +# cleanup: none +# === END CHECKS === + + +def test_critical_plan_is_balanced_and_digest_locked() -> None: + plan, artifact = load_frozen_plan() + counts = {ring: 0 for ring in ("phi", "psi", "omega")} + for case in plan.workload: + counts[case.expected_winner] += 1 + assert counts == {"phi": 6, "psi": 6, "omega": 6} + assert plan.digest == artifact["plan_digest"] + assert plan.minimum_target_accuracy == 0.75 + assert plan.minimum_target_advantage_vs_fallback == 0.05 + assert artifact["claim_rules"]["parity"] == "FALSIFIED for superiority only" + assert len(_canonical_digest(artifact)) == 64 + + +def test_result_digest_is_content_sensitive() -> None: + first = {"plan_digest": "a" * 64, "usefulness_status": "FALSIFIED"} + second = {"plan_digest": "a" * 64, "usefulness_status": "UNRESOLVED"} + assert _canonical_digest(first) == _canonical_digest(dict(first)) + assert _canonical_digest(first) != _canonical_digest(second) diff --git a/ptcna/tests/test_evaluation.py b/ptcna/tests/test_evaluation.py index b9f19b0..ff9ef99 100644 --- a/ptcna/tests/test_evaluation.py +++ b/ptcna/tests/test_evaluation.py @@ -33,6 +33,14 @@ # timeout: 30 # mutates: none # cleanup: none +# +# id: check_ptcna_comparator_failure_and_parity +# proves: ptcna_evaluation_propagates_backend_failure, ptcna_evaluation_verdict_uses_frozen_thresholds +# call: self::test_comparator_failure_is_unresolved_and_parity_is_not_superiority +# requires: python3 +# timeout: 30 +# mutates: none +# cleanup: none # === END CHECKS === @@ -72,14 +80,14 @@ def _plan(**changes) -> EvaluationPlan: EvaluationCase("two", "two", "psi"), ), "minimum_target_accuracy": 1.0, - "maximum_target_deficit_vs_fallback": 0.0, + "minimum_target_advantage_vs_fallback": 0.0, "training_epochs": 0, "reward_outcome": 1.0, "repetitions": 1, "max_training_steps": 0, "max_case_evaluations": 2, "max_seconds": 10.0, - "backend_error_status": FALSIFIED, + "target_backend_error_status": FALSIFIED, } values.update(changes) return EvaluationPlan(**values) @@ -112,9 +120,11 @@ def test_frozen_thresholds_produce_survival_and_falsification() -> None: falsified = evaluate( _plan(), target_factory=target_wrong, comparator_factory=fallback ) - assert survived.status == SURVIVED_NOT_PROVED + assert survived.usefulness_status == SURVIVED_NOT_PROVED + assert survived.superiority_status == SURVIVED_NOT_PROVED assert survived.target_accuracy == 1.0 - assert falsified.status == FALSIFIED + assert falsified.usefulness_status == FALSIFIED + assert falsified.superiority_status == FALSIFIED assert falsified.target_accuracy == 0.0 assert falsified.comparator_accuracy == 1.0 @@ -129,7 +139,7 @@ def test_frozen_thresholds_produce_survival_and_falsification() -> None: target_factory=learning_target, comparator_factory=learning_fallback, ) - assert learned.status == SURVIVED_NOT_PROVED + assert learned.usefulness_status == SURVIVED_NOT_PROVED assert learned.training_steps == 2 assert learned.target_accuracy == 1.0 @@ -140,11 +150,35 @@ def test_backend_failure_stops_with_preselected_status() -> None: FALLBACK_BACKEND, {"one": "phi", "two": "psi"} ) receipt = evaluate( - _plan(backend_error_status=FALSIFIED), + _plan(target_backend_error_status=FALSIFIED), target_factory=target, comparator_factory=fallback, ) - assert receipt.status == FALSIFIED + assert receipt.usefulness_status == FALSIFIED + assert receipt.superiority_status == "UNRESOLVED" assert receipt.training_steps == 0 assert receipt.case_evaluations == 0 - assert receipt.failure_reason == "backend_error:RuntimeError" + assert receipt.failure_reason == "target_backend_error:RuntimeError" + + +def test_comparator_failure_is_unresolved_and_parity_is_not_superiority() -> None: + target = lambda: _ScriptedBackend( + PTCNA_BACKEND, {"one": "phi", "two": "psi"} + ) + comparator_error = lambda: _ErrorBackend(FALLBACK_BACKEND, {}) + unresolved = evaluate( + _plan(), target_factory=target, comparator_factory=comparator_error + ) + assert unresolved.usefulness_status == "UNRESOLVED" + assert unresolved.superiority_status == "UNRESOLVED" + assert unresolved.failure_reason == "comparator_backend_error:RuntimeError" + + parity = evaluate( + _plan(minimum_target_advantage_vs_fallback=0.05), + target_factory=target, + comparator_factory=lambda: _ScriptedBackend( + FALLBACK_BACKEND, {"one": "phi", "two": "psi"} + ), + ) + assert parity.usefulness_status == SURVIVED_NOT_PROVED + assert parity.superiority_status == FALSIFIED diff --git a/ptcna_msdmd.ts b/ptcna_msdmd.ts index 5691aa3..d7ab14f 100644 --- a/ptcna_msdmd.ts +++ b/ptcna_msdmd.ts @@ -539,6 +539,67 @@ export default defineMsdmdCollection({ "file": "ptcna/core/prime_core/tests/test_ptca_core_stratified.py", "id": "check_prime_core_ucns_scope" }, + { + "block": "BOUNDARIES", + "fields": { + "admin_only": "false", + "auth_boundary": "none", + "network_boundary": "none", + "owner": "Erin Spencer", + "pii": "none", + "secrets": "none", + "since": "unreleased", + "storage_boundary": "write", + "summary": "reads the repository-owned frozen plan and writes one caller-selected local JSON result without network, authentication, secrets, or user data", + "user_data_boundary": "none" + }, + "file": "ptcna/critical_evaluation.py", + "id": "ptcna_critical_evaluation_local_receipt" + }, + { + "block": "CONTRACTS", + "fields": { + "class": "evidence", + "given": "the checked-in critical evaluation plan is loaded", + "then": "its canonical EvaluationPlan digest must equal the independently stored frozen digest" + }, + "file": "ptcna/critical_evaluation.py", + "id": "ptcna_critical_plan_digest_locked" + }, + { + "block": "CONTRACTS", + "fields": { + "class": "evidence", + "given": "the frozen plan completes or reaches a frozen failure rule", + "then": "the serialized result names the plan digest, separate claim verdicts, and its own canonical result digest" + }, + "file": "ptcna/critical_evaluation.py", + "id": "ptcna_critical_result_content_addressed" + }, + { + "block": "MODULE_BUILD", + "fields": { + "admin_only": "false", + "auth_boundary": "none", + "internal_surface": "_artifact_path, _canonical_digest", + "module_kind": "experiment", + "module_name": "critical_evaluation", + "network_boundary": "none", + "owner": "Erin Spencer", + "public_surface": "load_frozen_plan, execute_frozen_plan, main", + "requires": "ptcna_frozen_evaluation", + "rollback": "preserve plan and result receipts; remove executable wrapper without changing runtime", + "rollout": "execute only after the preregistration commit is merged", + "since": "unreleased", + "storage_boundary": "write", + "summary": "loads the immutable representative role-acquisition plan and seals its separate usefulness and superiority verdicts", + "tests": "ptcna/tests/test_critical_evaluation.py", + "unresolved": "outcome until the merged frozen plan is executed", + "user_data_boundary": "none" + }, + "file": "ptcna/critical_evaluation.py", + "id": "ptcna_critical_evaluation" + }, { "block": "BOUNDARIES", "fields": { @@ -571,7 +632,7 @@ export default defineMsdmdCollection({ "fields": { "class": "evidence", "given": "either backend errors before completing the frozen workload", - "then": "evaluation stops and records the plan's preselected backend_error_status before any repair or criterion change" + "then": "evaluation stops and records the plan's preselected target/comparator failure propagation before any repair or criterion change" }, "file": "ptcna/evaluation.py", "id": "ptcna_evaluation_propagates_backend_failure" @@ -581,7 +642,7 @@ export default defineMsdmdCollection({ "fields": { "class": "evidence", "given": "target and fallback complete the frozen workload", - "then": "training occurs before scoring and the terminal verdict is FALSIFIED or SURVIVED \u2014 not proved using only the plan's post-training accuracy and comparator-deficit thresholds" + "then": "training occurs before scoring and separate usefulness and superiority verdicts use only the frozen target-accuracy and target-advantage thresholds" }, "file": "ptcna/evaluation.py", "id": "ptcna_evaluation_verdict_uses_frozen_thresholds" @@ -1367,6 +1428,45 @@ export default defineMsdmdCollection({ "file": "ptcna/tests/test_contract_audit.py", "id": "check_contract_audit_complete_graph" }, + { + "block": "CHECKS", + "fields": { + "call": "self::test_critical_plan_is_balanced_and_digest_locked", + "cleanup": "none", + "mutates": "none", + "proves": "ptcna_critical_plan_digest_locked", + "requires": "python3", + "timeout": "30" + }, + "file": "ptcna/tests/test_critical_evaluation.py", + "id": "check_ptcna_critical_plan_digest" + }, + { + "block": "CHECKS", + "fields": { + "call": "self::test_result_digest_is_content_sensitive", + "cleanup": "none", + "mutates": "none", + "proves": "ptcna_critical_result_content_addressed", + "requires": "python3", + "timeout": "30" + }, + "file": "ptcna/tests/test_critical_evaluation.py", + "id": "check_ptcna_critical_result_digest" + }, + { + "block": "CHECKS", + "fields": { + "call": "self::test_comparator_failure_is_unresolved_and_parity_is_not_superiority", + "cleanup": "none", + "mutates": "none", + "proves": "ptcna_evaluation_propagates_backend_failure, ptcna_evaluation_verdict_uses_frozen_thresholds", + "requires": "python3", + "timeout": "30" + }, + "file": "ptcna/tests/test_evaluation.py", + "id": "check_ptcna_comparator_failure_and_parity" + }, { "block": "CHECKS", "fields": { @@ -1719,6 +1819,13 @@ export default defineMsdmdCollection({ "source_id": "prime_core_composition_runtime_boundary", "to": "Erin Spencer" }, + { + "from": "ptcna_critical_evaluation_local_receipt", + "kind": "owns", + "source_block": "BOUNDARIES", + "source_id": "ptcna_critical_evaluation_local_receipt", + "to": "Erin Spencer" + }, { "from": "ptcna_evaluation_local_boundary", "kind": "owns", @@ -2181,6 +2288,76 @@ export default defineMsdmdCollection({ "source_id": "check_prime_core_ucns_scope", "to": "python3" }, + { + "from": "check_ptcna_comparator_failure_and_parity", + "kind": "calls", + "source_block": "CHECKS", + "source_id": "check_ptcna_comparator_failure_and_parity", + "to": "self::test_comparator_failure_is_unresolved_and_parity_is_not_superiority" + }, + { + "from": "check_ptcna_comparator_failure_and_parity", + "kind": "claims_proves", + "source_block": "CHECKS", + "source_id": "check_ptcna_comparator_failure_and_parity", + "to": "ptcna_evaluation_propagates_backend_failure" + }, + { + "from": "check_ptcna_comparator_failure_and_parity", + "kind": "claims_proves", + "source_block": "CHECKS", + "source_id": "check_ptcna_comparator_failure_and_parity", + "to": "ptcna_evaluation_verdict_uses_frozen_thresholds" + }, + { + "from": "check_ptcna_comparator_failure_and_parity", + "kind": "requires", + "source_block": "CHECKS", + "source_id": "check_ptcna_comparator_failure_and_parity", + "to": "python3" + }, + { + "from": "check_ptcna_critical_plan_digest", + "kind": "calls", + "source_block": "CHECKS", + "source_id": "check_ptcna_critical_plan_digest", + "to": "self::test_critical_plan_is_balanced_and_digest_locked" + }, + { + "from": "check_ptcna_critical_plan_digest", + "kind": "claims_proves", + "source_block": "CHECKS", + "source_id": "check_ptcna_critical_plan_digest", + "to": "ptcna_critical_plan_digest_locked" + }, + { + "from": "check_ptcna_critical_plan_digest", + "kind": "requires", + "source_block": "CHECKS", + "source_id": "check_ptcna_critical_plan_digest", + "to": "python3" + }, + { + "from": "check_ptcna_critical_result_digest", + "kind": "calls", + "source_block": "CHECKS", + "source_id": "check_ptcna_critical_result_digest", + "to": "self::test_result_digest_is_content_sensitive" + }, + { + "from": "check_ptcna_critical_result_digest", + "kind": "claims_proves", + "source_block": "CHECKS", + "source_id": "check_ptcna_critical_result_digest", + "to": "ptcna_critical_result_content_addressed" + }, + { + "from": "check_ptcna_critical_result_digest", + "kind": "requires", + "source_block": "CHECKS", + "source_id": "check_ptcna_critical_result_digest", + "to": "python3" + }, { "from": "check_ptcna_explicit_failover", "kind": "calls", @@ -2965,6 +3142,20 @@ export default defineMsdmdCollection({ "source_id": "ptcna_contract_audit", "to": "vendored msdmd parser" }, + { + "from": "ptcna_critical_evaluation", + "kind": "owns", + "source_block": "MODULE_BUILD", + "source_id": "ptcna_critical_evaluation", + "to": "Erin Spencer" + }, + { + "from": "ptcna_critical_evaluation", + "kind": "requires", + "source_block": "MODULE_BUILD", + "source_id": "ptcna_critical_evaluation", + "to": "ptcna_frozen_evaluation" + }, { "from": "ptcna_fiq_host", "kind": "owns",