diff --git a/.github/workflows/render-report.yml b/.github/workflows/render-report.yml index 51c9f59..5503f1c 100644 --- a/.github/workflows/render-report.yml +++ b/.github/workflows/render-report.yml @@ -21,75 +21,22 @@ jobs: with: python-version: "3.13" - - name: Install Jupyter runtime - run: python -m pip install --upgrade pip jupyter papermill jsonschema - - - name: Ruff lint and format check (published core excluded — see ruff.toml) + - name: Install pinned gate runtime run: | - python -m pip install ruff - ruff check . - ruff format --check . - - - name: Run unit tests (includes frozen conformance vector) - run: PYTHONPATH=src python -m unittest discover -s tests -v - - - name: Verify experimental results artifact - run: PYTHONPATH=src python -m analysis.run_method_tranche --check - - - name: Verify Contract B v2 synthetic refusal artifact - run: PYTHONPATH=src python -m analysis.run_contract_v2 --check + python -m pip install --upgrade pip + python -m pip install '.[dev,report]' - - name: Build wheel - run: python -m pip wheel . --no-deps -w dist/ + - name: Read pinned Quarto version + id: quarto-version + run: echo "version=$(<.quarto-version)" >> "$GITHUB_OUTPUT" - name: Set up Quarto uses: quarto-dev/quarto-actions/setup@v2 + with: + version: "${{ steps.quarto-version.outputs.version }}" - - name: Validate sample evidence - run: make validate-report EVIDENCE=evidence/sample-lab-report.json - - - name: Render website - run: quarto render - - - name: Render sample report - run: | - python scripts/render_methodology_report.py \ - --evidence evidence/sample-lab-report.json \ - --report reports/methodology-report.qmd \ - --output-dir reports/rendered/sample - - - name: Collect sample report artifact - shell: bash - run: | - html_path="$(find reports/rendered/sample -name 'methodology-report.html' -print -quit)" - if [ -z "$html_path" ]; then - echo "rendered sample report not found under reports/rendered/sample" >&2 - exit 1 - fi - if [ "$html_path" != "reports/rendered/sample/methodology-report.html" ]; then - mv "$html_path" reports/rendered/sample/methodology-report.html - fi - - - name: Render campaign evidence reports - shell: bash - run: | - shopt -s nullglob globstar - campaign_files=(evidence/campaigns/**/evidence.json) - if [ ${#campaign_files[@]} -eq 0 ]; then - echo "No evidence/campaigns/**/evidence.json files present." - exit 0 - fi - - for evidence in "${campaign_files[@]}"; do - safe_name="${evidence#evidence/campaigns/}" - safe_name="${safe_name%/evidence.json}" - safe_name="${safe_name//\//-}" - out_dir="reports/rendered/campaigns/${safe_name}" - python scripts/render_methodology_report.py \ - --evidence "$evidence" \ - --report reports/methodology-report.qmd \ - --output-dir "$out_dir" - done + - name: Run canonical merge gate + run: bash scripts/run-merge-gate.sh - name: Upload rendered reports uses: actions/upload-artifact@v4 diff --git a/.gitignore b/.gitignore index f8fa75b..c0b8ab8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,11 +2,14 @@ __pycache__/ *.pyc -# Quarto render outputs +# Quarto render outputs and execution scratch .quarto/ _site/ +**/*.quarto_ipynb reports/*.html reports/rendered/ # OS cruft .DS_Store + +/.quarto/ diff --git a/.quarto-version b/.quarto-version new file mode 100644 index 0000000..a7c3730 --- /dev/null +++ b/.quarto-version @@ -0,0 +1 @@ +1.9.38 diff --git a/AGENTS.md b/AGENTS.md index 9b0f1ce..46e8ab0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -162,22 +162,19 @@ bootstrap, GLMM, Quarto). The public alias "Agentic-Coding Evaluation Lab" is fi ## Gate parity (local == CI) -The canonical merge gate — run from the repo root; local invocation equals CI, and -every stage must pass before merge: +The canonical merge gate — run from the repo root; CI invokes the same script and every stage must +pass before merge. Install the Python extras and the exact Quarto release named in +`.quarto-version`, then run: ``` -PYTHONPATH=src python -m unittest discover -s tests -v && PYTHONPATH=src python -m analysis.run_method_tranche --check && PYTHONPATH=src python -m analysis.run_contract_v2 --check +python -m pip install '.[dev,report]' && bash scripts/run-merge-gate.sh ``` -### Additional gates before commit - -In addition to the merge gate above, run these from the repo root; every one must -pass before a commit: - -- `PYTHONPATH=src python -m unittest discover -s tests -v` -- `make validate-report EVIDENCE=evidence/sample-lab-report.json` — and repeat - for each `evidence/campaigns/**/evidence.json` file -- `python -m pip wheel . --no-deps -w dist/` +The script first refuses a missing or mismatched Quarto version, then enforces Ruff lint/format +with the version pinned in `pyproject.toml`, unit tests, both +exact-result checks, wheel containment, sample and campaign evidence validation, the full Quarto +site render, and sample/campaign report rendering. Adding a CI stage requires adding it to this +script so local and CI stay identical. ## Safe vs held commands diff --git a/PLAN.md b/PLAN.md index ccfbf36..58c27d7 100644 --- a/PLAN.md +++ b/PLAN.md @@ -37,6 +37,11 @@ every commit is world-readable and every merge to `main` is a publication act. - No enforcing test, replicate policy, or minimum practically important benefit has been selected for the next contract. Powered promotion remains unavailable until that coupled decision is made from a complete package. +- The calibration protocol family now has separate E4 grader-validation, E1 + Lane A A/A session-position, and E2 operating-characteristic candidate SAPs. + They are not preregistered, authorize no observations, and select no powered + method. Their counts, thresholds, grids, and consequences remain review + candidates until separately frozen and registered. - Contract B `2.0.0-draft.1` is an immutable, repository-only normative draft beside v1. It binds synthetic registration, reveal, schedule, attempts, attrition, copied observation provenance, analysis, decision, and report @@ -49,7 +54,7 @@ every commit is world-readable and every merge to `main` is a publication act. unchanged core identity, runtime dependency, and synthetic conformance pair. This is a repository delivery contract, not a package release or operational adoption; the wheel-buildable 0.2.0 package contents remain unchanged. -- Its synthetic observation cardinality is one Aider-owner descriptor-validation +- Its synthetic observation cardinality is one editing-client-owner descriptor-validation receipt and one lab-owned completion receipt per attempt; one distinct gateway open receipt, zero or more gateway call receipts, and one distinct gateway closure receipt per invocation. Zero calls are admissible only for a retained @@ -78,9 +83,10 @@ open: 3. **Nuisance-boundary control.** Replace the current finite `0.01` grid judgment with a justified continuous-boundary argument or an explicitly conservative bound. -4. **Dependence.** Bound cross-task correlation induced by shared serving state, - session-by-arm interactions, batch composition, and prefix caching. Retain - session and task-pair identifiers so the assumption is auditable. +4. **Dependence.** For Lane A, use one task pair per fresh session and attempt to + falsify session-position invariance rather than estimate an intra-session + correlation. Retain actual session position and task-pair identifiers; treat + shared serving state and cache effects as explicit residual threats. 5. **Level-matched comparison.** Compare candidate procedures at the same demonstrated Type I level; do not attribute unmatched power gaps to one mechanism. diff --git a/README.md b/README.md index 7cdd43a..9814a77 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,8 @@ order to trust or challenge an evaluation. explicit decision. - No validated operating characteristics for the intended class-balanced design. - No operational registration signer, trusted timestamp path, or production owner-receipt join. -- No validated grader study, governed 40-task suite, A/A calibration, or powered campaign. +- No validated grader study, no public-safe validity evidence for the governed 40-task suite, no + A/A calibration, and no powered campaign. - No evidence that a candidate model or harness improves real task performance. ## Evidence and reproduction @@ -97,6 +98,7 @@ order to trust or challenge an evaluation. - [Refusal example](reports/contract-v2-not-evaluable.qmd) - [Identity-domain specification](identity-domain.qmd) - [Delivery manifest](evidence/contract-v2/delivery-manifest.v1.json) +- [Candidate calibration protocol family](docs/preregistrations/calibration-sap.md) - [Roadmap](PLAN.md) The repository contains two historical worked examples: @@ -110,17 +112,17 @@ historical artifacts, not evidence for model selection. ### Verify locally +Install the exact Quarto release named in `.quarto-version` (on macOS, +`brew install --cask quarto` installs that release when Homebrew's cask matches the pin). Then run: + ```bash -python -m pip install '.[dev]' -PYTHONPATH=src python -m unittest discover -s tests -v -PYTHONPATH=src python -m analysis.run_method_tranche --check -PYTHONPATH=src python -m analysis.run_contract_v2 --check -make validate-report EVIDENCE=evidence/sample-lab-report.json -python -m pip wheel . --no-deps -w dist/ +python -m pip install '.[dev,report]' +bash scripts/run-merge-gate.sh ``` -Quarto is also required to render the site and HTML reports. CI runs tests, exact-result checks, -wheel containment, report validation, and site rendering before publication. +The script refuses a missing or mismatched Quarto version. CI reads the same version pin and invokes +the same merge-gate script, covering Ruff, tests, exact-result checks, wheel containment, evidence +validation, and site/report rendering before publication. ## Technical details diff --git a/STATUS.md b/STATUS.md index 8ae07b0..879f640 100644 --- a/STATUS.md +++ b/STATUS.md @@ -16,9 +16,10 @@ ready for a powered scientific promotion decision or production use. | Item | Evidence | Status | |---|---|---| -| Public site | Render and Pages workflows passed for publication commit `3ceffb1` | published | +| Public site | Render run `29614595319` and Pages run `29614595347` passed for current published baseline `6aabf7f` | published | | Core version | `pyproject.toml` and frozen core identify `0.2.0`; wheel containment is tested | repository-versioned, not distributed — the version names code in this repository, not a release anyone can install from a registry | | Experimental calculations | Committed synthetic results regenerate byte-for-byte | decision input only — the numbers inform, but do not make, the pending method decision | +| Calibration SAP family | Separate E4, E1, and E2 candidate protocols define reviewable objectives, counts, failure rules, and claim boundaries | not preregistered; no observations authorized; numerical choices remain candidates | | Contract B v2 draft | Exact schema, verifier, renderer, safety scanner, fixture, report, and delivery manifest are digest-bound (every implementation byte is hash-pinned) | synthetic conformance only — the draft evidence contract is validated against invented data, not real campaigns | | Independent refusal | The public verifier derives `NOT_EVALUABLE` from the synthetic fixture and rejects mutations — it recomputes the result from frozen evidence bytes and refuses any altered copy | verified for the fixture | | Publication scanning | Evidence inputs are recursively scanned in the parameterized report path; Contract B checks its bound public payload separately | path-scoped; no whole-repository scan — leak scanning covers the report inputs, not every file in the repository | @@ -34,10 +35,10 @@ therefore identifies repository and wheel metadata, not a distributed release. | Operating characteristics | Type I error (how often the rule would be wrong), power (how often it would detect a real difference), dependence sensitivity, and feasibility are not established for the intended design | | Measurement validity | Grader validation, protected-task policy, and governed task selection are incomplete — the scoring instrument itself is not yet validated | | Operational provenance | Registration signing/timestamp verification and runtime owner receipts are not operational — runtime evidence is not yet signed and independently verified in production | -| Calibration | No accepted grader study, A/A independence/noise-floor study, or rule-calibration study exists — nothing yet shows the measurement apparatus is quiet and consistent enough to trust | +| Calibration | No accepted grader study, A/A session-position invariance study, or rule-calibration study exists — nothing yet shows the measurement apparatus is quiet and consistent enough to trust | | Real result | No powered, registered, scientifically admissible model comparison has been published | | Historical smoke reproducibility | Only summary statistics are public; raw per-attempt outcomes required to recompute the interval are absent — outside readers cannot independently re-derive it | -| Publication enforcement | CI passes on current `main`, but no GitHub ruleset or branch protection currently requires those checks — nothing mechanically blocks a merge that skips them | +| Publication enforcement | Active no-bypass ruleset `18954954` requires pull requests, signed commits, linear history, resolved review threads, and the strict `Render public evidence reports` check; approval count is zero under the disclosed solo-maintainer model | protected, but not independent review | ## Claim policy @@ -55,7 +56,7 @@ recomputation, and explicit refusal behavior. They do not support claims that: 1. Accept a scientific analysis plan and publish the corresponding versioned method contract. 2. Complete owner-issued runtime observations and operational registration verification. 3. Freeze the grader, task-selection record, task suite, assignment, retry, and attrition policies. -4. Run and review grader validation, A/A independence/noise-floor calibration, and operating- +4. Run and review grader validation, A/A session-position invariance calibration, and operating- characteristic analysis. 5. Register and execute the first powered local comparison. 6. Independently recompute, sanitize, review, and publish the result, including a no-go or @@ -72,5 +73,5 @@ conformance vector are unchanged. ## Provenance -- Status date: 2026-07-14 -- Audited public baseline: `3ceffb1474146c8b462af11112faf4c50b547ce8` +- Status date: 2026-07-17 +- Baseline reviewed for this candidate update: `6aabf7f32e3d824ee63f2eef3a761f29be4e2d4c` diff --git a/build_backend.py b/build_backend.py index 6bbcc57..bdff4cf 100644 --- a/build_backend.py +++ b/build_backend.py @@ -7,15 +7,19 @@ import hashlib import io import tarfile +import tomllib import zipfile from pathlib import Path from typing import Any -NAME = "eval-lab-methodology" -NORMALIZED_NAME = "eval_lab_methodology" -VERSION = "0.2.0" -DIST_INFO = f"{NORMALIZED_NAME}-{VERSION}.dist-info" ROOT = Path(__file__).resolve().parent +PROJECT = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))[ + "project" +] +NAME = PROJECT["name"] +NORMALIZED_NAME = NAME.replace("-", "_") +VERSION = PROJECT["version"] +DIST_INFO = f"{NORMALIZED_NAME}-{VERSION}.dist-info" PACKAGE_ROOT = ROOT / "src" / "eval_lab_methodology" @@ -118,24 +122,25 @@ def _is_bytecode(path: Path) -> bool: def _metadata() -> str: - return "\n".join( - [ - "Metadata-Version: 2.3", - f"Name: {NAME}", - f"Version: {VERSION}", - "Summary: Public statistical and reporting core for the Agentic-Coding Evaluation Lab methodology.", - "Author: Verlyn Nash", - "License-Expression: Apache-2.0", - "Requires-Python: >=3.11", - "Provides-Extra: glmm", - "Requires-Dist: pandas>=2.0; extra == 'glmm'", - "Requires-Dist: statsmodels>=0.14; extra == 'glmm'", - "Provides-Extra: dev", - "Requires-Dist: jsonschema>=4.22; extra == 'dev'", - "Requires-Dist: pytest>=8.0; extra == 'dev'", - "", - ] - ) + lines = [ + "Metadata-Version: 2.3", + f"Name: {NAME}", + f"Version: {VERSION}", + f"Summary: {PROJECT['description']}", + f"Author: {PROJECT['authors'][0]['name']}", + f"License-Expression: {PROJECT['license']}", + f"Requires-Python: {PROJECT['requires-python']}", + ] + for requirement in PROJECT.get("dependencies", []): + lines.append(f"Requires-Dist: {requirement}") + for extra, requirements in PROJECT.get("optional-dependencies", {}).items(): + lines.append(f"Provides-Extra: {extra}") + lines.extend( + f"Requires-Dist: {requirement}; extra == '{extra}'" + for requirement in requirements + ) + lines.append("") + return "\n".join(lines) def _wheel_metadata() -> str: diff --git a/docs/preregistrations/calibration-sap.md b/docs/preregistrations/calibration-sap.md index 260640d..480f4bd 100644 --- a/docs/preregistrations/calibration-sap.md +++ b/docs/preregistrations/calibration-sap.md @@ -1,62 +1,44 @@ -# Calibration Statistical Analysis Plan (SAP) - -## 1. Metadata -- **Title**: Phase 12 Lane A Instrument Calibration Preregistration -- **Date**: 2026-07-16 -- **Version**: 1.0.0 -- **Status**: Preregistered Draft - ---- - -## 2. E4 Grader Validation Protocol -The objective of E4 is to measure the agreement between the deterministic machine grader ($G_m$) and the independent human gold label ($G_h$). - -### Metrics -We will compute the following metrics over a class-balanced validation corpus ($N \ge 40$ attempts representing all four classes and failure modes): - -* **Sensitivity (True Positive Rate)**: - $$\text{Sensitivity} = \frac{TP}{TP + FN}$$ - *Where $TP$ is the number of attempts graded as success by both human and machine, and $FN$ is the number of attempts graded as success by human but failure by machine.* - -* **Specificity (True Negative Rate)**: - $$\text{Specificity} = \frac{TN}{TN + FP}$$ - *Where $TN$ is graded as failure by both, and $FP$ is graded as failure by human but success by machine.* - -* **Positive Predictive Value (PPV)**: - $$\text{PPV} = \frac{TP}{TP + FP}$$ - -* **Negative Predictive Value (NPV)**: - $$\text{NPV} = \frac{TN}{TN + FN}$$ - -### Validation Thresholds -The instrument is declared valid for calibration only if: -* $\text{Sensitivity} \ge 0.90$ -* $\text{Specificity} \ge 0.95$ - ---- - -## 3. E1 A/A Independence Study -The objective of E1 is to estimate the within-session correlation coefficient ($\rho$) and verify that local hardware and cache conditions do not introduce systematic bias between experimental arms. - -### Design -* **Arms**: Arm 1 (A) and Arm 2 (A') run identical model digests (`qwen3.5:9b`). -* **Sample Size**: 3 serial replicates of the frozen 40-task suite per arm (240 total attempts). -* **Null Hypothesis**: $H_0: \rho = 0$ (attempts are independent). -* **Test Statistic**: Pearson correlation coefficient and Ljung-Box test for serial autocorrelation on latency and success vectors. - ---- - -## 4. E2 Operating Characteristics Simulation -E2 evaluates the provisional decision rule and estimates statistical power under varying effect sizes. - -### Simulation Inputs -* Empirical success rates and transition matrices from E1. -* Measurement error rates from E4. -* Simulation runs ($B = 10,000$) to evaluate the Type I error rate ($\alpha$) and statistical power ($1-\beta$) under candidate sample sizes ($N$) and Minimum Practical Improvement Boundaries (MPIB). - ---- - -## 5. Attrition & Missingness Handling -Every initiated attempt must be accounted for in the attrition table. -* **Timeout/OOM/Harness Error**: Evaluated as `NOT_EVALUABLE` and excluded from the capability pass-rate calculations, but documented in the attrition table. -* **Transient Network Error**: Subject to a maximum of 1 automatic retry; the retry carries its own unique attempt index and receipts. +# Lane A calibration protocol family + +## Metadata + +- **Date:** 2026-07-17 +- **Version:** 1.1.0-candidate.1 +- **Status:** Candidate protocol — not preregistered; no observations authorized. + +## Purpose + +This document is the index and common claim boundary for three separate candidate statistical +analysis plans: + +1. [E4 grader validation](e4-grader-validation-sap.md) +2. [E1 Lane A A/A session-position study](e1-lane-a-aa-sap.md) +3. [E2 operating-characteristics simulation](e2-operating-characteristics-sap.md) + +The former umbrella draft combined three different studies and incorrectly described the E1 +objective as estimation of within-session correlation. The replacement family uses Lane A `m = 1` +by construction: one task pair per fresh session. E1 is therefore a determinism/falsification study, +not a correlation-estimation study. + +## Common rules + +- E4, E1, and E2 require separate reviewed registrations and separate result artifacts. +- Planned values in these documents are design candidates, not produced observations. +- Every initiated E1 attempt is retained as exactly one terminal attempt row. Failure, timeout, + resource exhaustion, receipt failure, and harness failure are outcomes; none is silently dropped. +- No whole-attempt automatic retry is allowed. A separately authorized rerun must use a new attempt + ordinal and a new registration rather than replacing the original record. +- Each gateway call has a planned provider retry limit of zero. A client invocation may contain + multiple disclosed calls; all calls must be included in its closure record. +- No fallback model, direct provider path, or unregistered schedule substitution is allowed. +- A missing registration, byte binding, assignment row, owner receipt, invocation closure, terminal + attempt row, or required analysis artifact makes the affected study `NOT_EVALUABLE`. +- Calibration results cannot promote a system, select a powered method, or settle a minimum + practically important benefit. Those require a later coupled ruling and a new protocol. + +## Current readiness + +These documents are reviewable design inputs only. Before any study can be registered, reviewers +must resolve the candidate decisions called out in each SAP, freeze the machine-readable inputs and +implementations by digest, approve the evidence/report contracts, and establish operational +registration and observation authority. diff --git a/docs/preregistrations/e1-lane-a-aa-sap.md b/docs/preregistrations/e1-lane-a-aa-sap.md new file mode 100644 index 0000000..1ea525b --- /dev/null +++ b/docs/preregistrations/e1-lane-a-aa-sap.md @@ -0,0 +1,108 @@ +# E1 Lane A A/A session-position statistical analysis plan + +## Metadata and claim boundary + +- **Date:** 2026-07-17 +- **Version:** 0.1.0-candidate.1 +- **Status:** Candidate protocol — not preregistered; no observations authorized. +- **Study type:** Local-lane calibration; structurally non-promotable. + +All counts, seeds, and rules below are **planned candidate values**. They are not observations and +do not establish a powered-study replicate policy. + +## Objective + +E1 attempts to falsify exact outcome invariance when two blinded arms resolve to the same content +digest and task pairs are isolated by fresh sessions. Lane A uses **`m = 1`**: one task pair per +fresh session. The design removes within-session clustering from the later task-level comparison by +construction; E1 does not estimate an intra-session correlation and does not test a null hypothesis +that such a correlation is zero. + +E1 reports whether task success and exact outcome digests remain identical across A/A arms and +registered session positions. Latency and resource measures are diagnostics only and never turn a +divergent outcome into an invariant one. + +## Planned design and counts + +- 40 registered task pairs, with 10 in each of four fixed task classes. +- 3 rounds, each containing one fresh session for every task pair. +- 120 sessions total: `40 task pairs x 3 rounds`. +- 2 blinded arm attempts per session, yielding 240 terminal arm attempts total. +- Each arm attempt has a separate editing-client descriptor and a separate gateway invocation with + open, zero-or-more call, and closure receipts. +- Both arms must bind the same resolved content digest, decoding configuration, task bytes, harness, + grader, prompt, identity-domain preimage, and resource policy. + +The number of model calls is not fixed at 240. A client invocation may make multiple gateway calls, +all of which must appear in invocation closure. Zero calls are valid only for a retained terminal +pre-dispatch failure. + +## Planned assignment schedule + +The candidate schedule seed is the UTF-8 string `lane-a-aa-schedule-v1`. Sort registered opaque task +identifiers by `SHA-256(seed || NUL || task_id)`. Use that base order in round 1 and rotate it by 13 +and 27 positions in rounds 2 and 3. The machine-readable schedule must bind the resulting task, +round, planned-position, and arm-order rows before registration. + +Within each round, arm order is determined by the parity of `round_index + planned_position`, where +round indices are 0, 1, and 2 and positions are 1 through 40. This yields exactly 20 sessions in each +arm order per round and 60 in each arm order overall. + +The execution record must also store the **actual chronological session position**. Analysis uses +actual position and reports any deviation from the registered order; a synthetic early/middle/late +label is not a substitute. + +## Session isolation and execution + +Before each task-pair session, create isolated workspaces and fresh client processes for both arms. +Each arm gets its own invocation and workspace; neither arm may read the other's artifacts. Close +both invocations and emit both terminal attempt rows before beginning the next session. Record +machine-load, model-residency, cache, process, harness, and verification-subprocess diagnostics that +the registered observation contract can actually support. Do not claim an engine restart unless it +is observed and receipt-bound. + +The provider retry limit is zero for every gateway call. There is no automatic whole-attempt retry. +Client-generated additional calls are disclosed within the same invocation and do not create a new +scientific attempt. Every failure remains in the attempt ledger. + +## Analysis and falsification rule + +For each task and round, compare the two arms' `task_success` and exact `outcome_digest`. For each +task across its three actual positions, compare the same fingerprint. Report: + +- cross-arm and cross-position divergence counts and exact task/round locations; +- terminal status and failure-mode counts by arm order, round, class, and actual position; +- latency and resource summaries by the same strata, labeled diagnostic; and +- all schedule deviations, receipt failures, and incomplete joins. + +The candidate E1 result has four states, applied in the following precedence order: + +- `INVARIANCE_NOT_REFUTED` when all 120 sessions and 240 arm attempts have complete terminal + accounting, every required owner join and digest binding validates, no unplanned terminal + apparatus failure occurs, and there is zero cross-arm and zero cross-position divergence; +- `INVARIANCE_REFUTED` when the evidence is complete and valid but at least one fingerprint + diverges; +- `APPARATUS_NOT_ADMISSIBLE` when fingerprints do not diverge but complete, valid evidence contains + an unplanned timeout, resource, harness, or client failure; or +- `NOT_EVALUABLE` when missing or invalid registration, schedule, binding, receipt, closure, or + terminal accounting prevents the falsification analysis from being trusted. + +Thus a valid negative observation is reported as an invariance falsification or an apparatus +admission failure, not discarded as missingness. +This exact-invariance rule is a candidate requiring formal review; a clean result does not prove +invariance outside the registered apparatus and makes no population-level claim about a correlation +parameter. + +## Hard stops and consequences + +Stop dispatch on a content-digest mismatch, fallback, direct unregistered transport, broken receipt +chain, schedule mutation, evidence-integrity failure, or loss of session isolation. Already initiated +attempts remain reportable terminal rows. A validly observed timeout, resource exhaustion, harness +error, or client error is not dropped or replaced and blocks apparatus admission; absent fingerprint +divergence it yields `APPARATUS_NOT_ADMISSIBLE`. A broken evidence chain yields `NOT_EVALUABLE`. + +An evaluable E1 result is evidence only about the frozen local apparatus and schedule. It cannot +validate a remote lane, select an enforcing test, define the powered replicate unit, or authorize +promotion. +Before registration, reviewers must freeze the executable schedule, wall-time bound, zero-dollar +resource budget, stop implementation, evidence schema, and report schema by exact digest. diff --git a/docs/preregistrations/e2-operating-characteristics-sap.md b/docs/preregistrations/e2-operating-characteristics-sap.md new file mode 100644 index 0000000..ccc4a9f --- /dev/null +++ b/docs/preregistrations/e2-operating-characteristics-sap.md @@ -0,0 +1,76 @@ +# E2 operating-characteristics statistical analysis plan + +## Metadata and claim boundary + +- **Date:** 2026-07-17 +- **Version:** 0.1.0-candidate.1 +- **Status:** Candidate protocol — not preregistered; no observations authorized. +- **Study type:** Deterministic synthetic simulation; no model or provider execution. + +Every value below is a **planned candidate value**. E2 compares candidate decision rules; it does +not select one, settle R1/R2/R4, or produce model-performance evidence. + +## Objective + +E2 estimates operating characteristics of exact, hash-bound candidate rule implementations under +the finite 40-task, four-class design. It evaluates Type I error, power, missingness, grader error, +heterogeneity, and residual apparatus sensitivity before any powered protocol can be considered. + +No rule is currently accepted. E2 cannot run until the candidate implementations, estimands, +nuisance handling, and simulation distributions are frozen by exact digest. A result for an +unfrozen or approximate stand-in does not apply to the later powered rule. + +## Candidate scenario grid + +The proposed grid crosses: + +- total task-pair counts `N` in `{20, 40, 60}` with equal planned class weighting; +- incumbent success probabilities in `{0.30, 0.50, 0.70, 0.90}`; +- mean candidate-minus-incumbent effects in `{0.00, 0.025, 0.05, 0.10, 0.20}`; +- effect patterns: homogeneous, balanced opposing class effects, sparse benefit, and one-class harm; +- terminal missing/failure rates in `{0.00, 0.01, 0.05, 0.10}` under registered arm-symmetric and + arm-asymmetric mechanisms; +- grader-error matrices from the accepted E4 result plus registered boundary sensitivity cases; and +- exact apparatus invariance plus registered divergence cases derived from the accepted E1 result. + +Impossible probability combinations are omitted by an explicit deterministic rule and listed in +the scenario manifest. Each retained cell receives 10,000 planned simulation draws. The root seed is +the UTF-8 string `lane-a-e2-oc-v1`; a cell seed is derived from the root seed and canonical cell JSON +using SHA-256. Simulation order cannot affect a cell's draws. + +The grid, probability model, paired-outcome construction, missingness mechanism, and candidate-rule +implementations require formal methodology review before registration. + +## Outputs + +For every candidate rule and scenario cell, report rejection count, empirical rejection rate, Monte +Carlo standard error, and an exact binomial confidence interval. Label null cells as Type I error and +alternative cells as power. Also report refusal/`NOT_EVALUABLE` frequency, decision frequency, +class-specific error, and sensitivity to grader and apparatus violations. + +All cell-level records, aggregate tables, code digests, dependency lock, scenario manifest, seeds, +and report bytes must be committed as a deterministic recomputation packet by a future registered +E2 study. That packet must regenerate exactly under the registered environment. + +## Candidate admission criterion + +Using a one-sided nominal level of 0.05, a candidate rule is eligible for the later coupled ruling +only if the one-sided 95% exact-binomial upper confidence bound on its Type I error is at most 0.06 +in **every** registered null cell. Power has no pass threshold in this candidate; it is reported with +feasibility and practical-benefit consequences. + +The 0.06 bound, grid, draw count, and interval choice remain formal decisions. Eligibility is not +selection: the later ruling must consider validity assumptions, power, feasibility, E4/E1 results, +and practical benefit together. + +## Missing inputs and hard stops + +E2 is `NOT_EVALUABLE` if an accepted E4/E1 input is required but absent, a rule or grid digest does +not match registration, any registered cell is skipped, the draw count is incomplete, an output +cannot be exactly regenerated, or a producer claim differs from independent recomputation. Stop on +the first integrity mismatch and preserve the partial diagnostic separately from the registered +result. + +Before registration, reviewers must freeze the candidate rule set, complete bounded-mean +alternative, continuous nuisance-boundary treatment, cell generator, draw count, error criterion, +compute/wall-time budget, output schema, and independent recomputation command. diff --git a/docs/preregistrations/e4-grader-validation-sap.md b/docs/preregistrations/e4-grader-validation-sap.md new file mode 100644 index 0000000..3da3bf7 --- /dev/null +++ b/docs/preregistrations/e4-grader-validation-sap.md @@ -0,0 +1,83 @@ +# E4 grader-validation statistical analysis plan + +## Metadata and claim boundary + +- **Date:** 2026-07-17 +- **Version:** 0.1.0-candidate.1 +- **Status:** Candidate protocol — not preregistered; no observations authorized. +- **Study type:** Authored-fixture instrument validation; no model or provider execution. + +All counts and thresholds below are **planned candidate values**. They are neither accepted design +parameters nor produced results. + +## Objective + +E4 tests construct validity of the deterministic editing grader against an independently authored +gold label. It does not test whether a deterministic function agrees with itself. Re-executed +score-path repeatability belongs to E1's outcome-digest checks. + +The primary question is whether machine success/failure agrees sufficiently with the adjudicated +human interpretation of the task contract to admit the instrument for calibration studies. Passing +E4 would not validate a powered comparison or establish general validity outside the frozen task +suite and grader version. + +## Planned corpus + +- 160 authored output records: 40 from each of four registered task classes. +- Within each class, 20 records are authored to satisfy the task contract and 20 are authored not to + satisfy it, for a planned 80/80 gold-label balance. +- Non-success records cover registered failure strata, including incomplete edits, out-of-scope + edits, verification failure, setup failure, timeout, and malformed output where applicable. +- Corpus records, task contracts, expected labels, failure strata, and the machine grader are frozen + by exact digest before either reviewer receives an assignment. +- Corpus construction cannot use outputs from the later E1 study or any powered campaign. + +The corpus size and strata are candidates that require methodology review before registration. + +## Independent labels and adjudication + +Two qualified reviewers independently label every record without seeing the machine grade, the +other reviewer's label, or a success/non-success target. Reviewer order is randomized. Reviewers +record a binary label, rationale code, and ambiguity flag. + +Agreement between reviewers becomes the provisional gold label. A disagreement or any ambiguity +flag goes to a third adjudicator, who sees the task contract and both rationales but remains blinded +to the machine grade. The adjudicated label is the analysis gold label. Reviewer identities, +qualification criteria, conflicts, assignments, and adjudications must be retained in the private +study record; only a sanitized role-level report is public. + +## Analysis + +Using the adjudicated human label as truth and machine success as the prediction, report `TP`, `TN`, +`FP`, and `FN`, then compute: + +\[ +\mathrm{sensitivity}=\frac{TP}{TP+FN},\qquad +\mathrm{specificity}=\frac{TN}{TN+FP}. +\] + +Also report positive predictive value, negative predictive value, class-stratified confusion +matrices, failure-stratum errors, raw reviewer agreement, and Cohen's kappa. Sensitivity and +specificity receive two-sided 95% Wilson intervals. Reviewer-agreement measures are disclosures, +not substitute admission criteria. + +## Candidate admission rule + +The instrument is only a **candidate for calibration admission** when all of these planned criteria +hold: + +- point sensitivity is at least 0.90; +- point specificity is at least 0.95; +- the two-sided 95% Wilson lower bound is at least 0.80 for both sensitivity and specificity; +- all 160 records have complete independent labels and any required adjudication; and +- no corpus, grader, task-contract, blinding, or byte-binding violation occurred. + +These numerical thresholds remain decisions for formal review. Even if achieved, the registered +E4 result must still be accepted explicitly; this draft cannot admit the instrument by itself. + +## Missingness and hard stops + +Any missing label, lost rationale, failed blinding check, post-assignment corpus change, or absent +digest makes E4 `NOT_EVALUABLE`. Records are not replaced after review begins. E4 stops immediately +on unblinding or evidence-integrity failure. The review schedule, reviewer-hour budget, conflict +rule, and exact public report schema must be frozen before registration. diff --git a/pyproject.toml b/pyproject.toml index eee44e2..edb23e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,11 @@ glmm = [ "statsmodels>=0.14" ] dev = [ - "jsonschema>=4.22", - "pytest>=8.0" + "jsonschema==4.26.0", + "pytest==9.1.1", + "ruff==0.15.10" +] +report = [ + "jupyter==1.1.1", + "papermill==2.7.0" ] diff --git a/scripts/run-merge-gate.sh b/scripts/run-merge-gate.sh new file mode 100644 index 0000000..ac658a1 --- /dev/null +++ b/scripts/run-merge-gate.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +required_quarto_version="$(tr -d '[:space:]' < .quarto-version)" +if ! command -v quarto >/dev/null 2>&1; then + echo "Quarto ${required_quarto_version} is required; see README.md" >&2 + exit 1 +fi +actual_quarto_version="$(quarto --version)" +if [[ "${actual_quarto_version}" != "${required_quarto_version}" ]]; then + echo "Quarto ${required_quarto_version} is required; found ${actual_quarto_version}" >&2 + exit 1 +fi + +wheel_dir="$(mktemp -d)" +trap 'rm -rf -- "${wheel_dir}"' EXIT + +ruff check . +ruff format --check . +PYTHONPATH=src python -m unittest discover -s tests -v +PYTHONPATH=src python -m analysis.run_method_tranche --check +PYTHONPATH=src python -m analysis.run_contract_v2 --check +python -m pip wheel . --no-deps -w "${wheel_dir}" +make validate-report EVIDENCE=evidence/sample-lab-report.json +quarto render +python scripts/render_methodology_report.py \ + --evidence evidence/sample-lab-report.json \ + --report reports/methodology-report.qmd \ + --output-dir reports/rendered/sample + +html_path="$(find reports/rendered/sample -name 'methodology-report.html' -print -quit)" +if [[ -z "${html_path}" ]]; then + echo "rendered sample report not found under reports/rendered/sample" >&2 + exit 1 +fi +if [[ "${html_path}" != "reports/rendered/sample/methodology-report.html" ]]; then + mv "${html_path}" reports/rendered/sample/methodology-report.html +fi + +shopt -s nullglob globstar +campaign_files=(evidence/campaigns/**/evidence.json) +for evidence in "${campaign_files[@]}"; do + safe_name="${evidence#evidence/campaigns/}" + safe_name="${safe_name%/evidence.json}" + safe_name="${safe_name//\//-}" + output_dir="reports/rendered/campaigns/${safe_name}" + make validate-report EVIDENCE="${evidence}" + python scripts/render_methodology_report.py \ + --evidence "${evidence}" \ + --report reports/methodology-report.qmd \ + --output-dir "${output_dir}" +done diff --git a/tests/test_calibration_saps.py b/tests/test_calibration_saps.py new file mode 100644 index 0000000..8a26816 --- /dev/null +++ b/tests/test_calibration_saps.py @@ -0,0 +1,58 @@ +"""Public-safety and claim-boundary checks for candidate calibration SAPs.""" + +from __future__ import annotations + +import unittest +from pathlib import Path + +from scripts.publication_safety import assert_publication_content_is_safe + +ROOT = Path(__file__).resolve().parents[1] +SAP_DIR = ROOT / "docs" / "preregistrations" +SAP_FILES = ( + SAP_DIR / "calibration-sap.md", + SAP_DIR / "e4-grader-validation-sap.md", + SAP_DIR / "e1-lane-a-aa-sap.md", + SAP_DIR / "e2-operating-characteristics-sap.md", +) + + +class CalibrationSapTests(unittest.TestCase): + def test_saps_are_explicitly_unregistered_and_public_safe(self) -> None: + for path in SAP_FILES: + with self.subTest(path=path.name): + text = path.read_text(encoding="utf-8") + self.assertIn( + "Status:** Candidate protocol — not preregistered; no observations authorized.", + text, + ) + self.assertNotIn("Status:** Preregistered Draft", text) + assert_publication_content_is_safe(text) + + def test_e1_uses_m_one_and_fixed_terminal_counts(self) -> None: + text = (SAP_DIR / "e1-lane-a-aa-sap.md").read_text(encoding="utf-8") + self.assertIn("`m = 1`", text) + self.assertIn("120 sessions total", text) + self.assertIn("240 terminal arm attempts total", text) + self.assertIn("actual chronological session position", text) + self.assertIn("`INVARIANCE_NOT_REFUTED`", text) + self.assertIn("`INVARIANCE_REFUTED`", text) + self.assertIn("`APPARATUS_NOT_ADMISSIBLE`", text) + self.assertIn("`NOT_EVALUABLE`", text) + self.assertNotIn("objective is to estimate", text.lower()) + self.assertNotIn("pearson correlation", text.lower()) + + def test_e4_and_e2_numbers_are_labeled_candidate(self) -> None: + e4 = (SAP_DIR / "e4-grader-validation-sap.md").read_text(encoding="utf-8") + e2 = (SAP_DIR / "e2-operating-characteristics-sap.md").read_text( + encoding="utf-8" + ) + self.assertIn("160 authored output records", e4) + self.assertIn("planned candidate values", e4) + self.assertIn("10,000 planned simulation draws", e2) + self.assertIn("planned candidate value", e2) + self.assertIn("must be committed", e2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_experimental_module_containment.py b/tests/test_experimental_module_containment.py index cef0ebf..c3cf0c0 100644 --- a/tests/test_experimental_module_containment.py +++ b/tests/test_experimental_module_containment.py @@ -4,6 +4,7 @@ import sys import tempfile +import tomllib import unittest import zipfile from pathlib import Path @@ -47,6 +48,16 @@ def test_versioned_wheel_excludes_experimental_modules(self) -> None: ).decode("utf-8") self.assertIn("Version: 0.2.0\n", metadata) + project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))[ + "project" + ] + for extra, requirements in project["optional-dependencies"].items(): + self.assertIn(f"Provides-Extra: {extra}\n", metadata) + for requirement in requirements: + self.assertIn( + f"Requires-Dist: {requirement}; extra == '{extra}'\n", + metadata, + ) for module in EXPERIMENTAL_MODULES: self.assertNotIn(f"eval_lab_methodology/{module}", names) self.assertFalse(any("_method_tranche" in name for name in names))