docs: define candidate calibration protocols

.github/workflows/render-report.yml

@@ -21,75 +21,22 @@ jobs:
         with:
           python-version: "3.13"
 
-      - name: Install Jupyter runtime
-        run: python -m pip install --upgrade pip jupyter papermill jsonschema
-
-      - name: Ruff lint and format check (published core excluded — see ruff.toml)
+      - name: Install pinned gate runtime
         run: |
-          python -m pip install ruff
-          ruff check .
-          ruff format --check .
-
-      - name: Run unit tests (includes frozen conformance vector)
-        run: PYTHONPATH=src python -m unittest discover -s tests -v
-
-      - name: Verify experimental results artifact
-        run: PYTHONPATH=src python -m analysis.run_method_tranche --check
-
-      - name: Verify Contract B v2 synthetic refusal artifact
-        run: PYTHONPATH=src python -m analysis.run_contract_v2 --check
+          python -m pip install --upgrade pip
+          python -m pip install '.[dev,report]'
 
-      - name: Build wheel
-        run: python -m pip wheel . --no-deps -w dist/
+      - name: Read pinned Quarto version
+        id: quarto-version
+        run: echo "version=$(<.quarto-version)" >> "$GITHUB_OUTPUT"
 
       - name: Set up Quarto
         uses: quarto-dev/quarto-actions/setup@v2
+        with:
+          version: "${{ steps.quarto-version.outputs.version }}"
 
-      - name: Validate sample evidence
-        run: make validate-report EVIDENCE=evidence/sample-lab-report.json
-
-      - name: Render website
-        run: quarto render
-
-      - name: Render sample report
-        run: |
-          python scripts/render_methodology_report.py \
-            --evidence evidence/sample-lab-report.json \
-            --report reports/methodology-report.qmd \
-            --output-dir reports/rendered/sample
-
-      - name: Collect sample report artifact
-        shell: bash
-        run: |
-          html_path="$(find reports/rendered/sample -name 'methodology-report.html' -print -quit)"
-          if [ -z "$html_path" ]; then
-            echo "rendered sample report not found under reports/rendered/sample" >&2
-            exit 1
-          fi
-          if [ "$html_path" != "reports/rendered/sample/methodology-report.html" ]; then
-            mv "$html_path" reports/rendered/sample/methodology-report.html
-          fi
-
-      - name: Render campaign evidence reports
-        shell: bash
-        run: |
-          shopt -s nullglob globstar
-          campaign_files=(evidence/campaigns/**/evidence.json)
-          if [ ${#campaign_files[@]} -eq 0 ]; then
-            echo "No evidence/campaigns/**/evidence.json files present."
-            exit 0
-          fi
-
-          for evidence in "${campaign_files[@]}"; do
-            safe_name="${evidence#evidence/campaigns/}"
-            safe_name="${safe_name%/evidence.json}"
-            safe_name="${safe_name//\//-}"
-            out_dir="reports/rendered/campaigns/${safe_name}"
-            python scripts/render_methodology_report.py \
-              --evidence "$evidence" \
-              --report reports/methodology-report.qmd \
-              --output-dir "$out_dir"
-          done
+      - name: Run canonical merge gate
+        run: bash scripts/run-merge-gate.sh
 
       - name: Upload rendered reports
         uses: actions/upload-artifact@v4

.gitignore

@@ -2,11 +2,14 @@
 __pycache__/
 *.pyc
 
-# Quarto render outputs
+# Quarto render outputs and execution scratch
 .quarto/
 _site/
+**/*.quarto_ipynb
 reports/*.html
 reports/rendered/
 
 # OS cruft
 .DS_Store
+
+/.quarto/

.quarto-version

@@ -0,0 +1 @@
+1.9.38

AGENTS.md

@@ -162,22 +162,19 @@ bootstrap, GLMM, Quarto). The public alias "Agentic-Coding Evaluation Lab" is fi
 
 ## Gate parity (local == CI)
 
-The canonical merge gate — run from the repo root; local invocation equals CI, and
-every stage must pass before merge:
+The canonical merge gate — run from the repo root; CI invokes the same script and every stage must
+pass before merge. Install the Python extras and the exact Quarto release named in
+`.quarto-version`, then run:
 
 ```
-PYTHONPATH=src python -m unittest discover -s tests -v && PYTHONPATH=src python -m analysis.run_method_tranche --check && PYTHONPATH=src python -m analysis.run_contract_v2 --check
+python -m pip install '.[dev,report]' && bash scripts/run-merge-gate.sh
 ```
 
-### Additional gates before commit
-
-In addition to the merge gate above, run these from the repo root; every one must
-pass before a commit:
-
-- `PYTHONPATH=src python -m unittest discover -s tests -v`
-- `make validate-report EVIDENCE=evidence/sample-lab-report.json` — and repeat
-  for each `evidence/campaigns/**/evidence.json` file
-- `python -m pip wheel . --no-deps -w dist/`
+The script first refuses a missing or mismatched Quarto version, then enforces Ruff lint/format
+with the version pinned in `pyproject.toml`, unit tests, both
+exact-result checks, wheel containment, sample and campaign evidence validation, the full Quarto
+site render, and sample/campaign report rendering. Adding a CI stage requires adding it to this
+script so local and CI stay identical.
 
 ## Safe vs held commands
 

PLAN.md

@@ -37,6 +37,11 @@ every commit is world-readable and every merge to `main` is a publication act.
 - No enforcing test, replicate policy, or minimum practically important
   benefit has been selected for the next contract. Powered promotion remains
   unavailable until that coupled decision is made from a complete package.
+- The calibration protocol family now has separate E4 grader-validation, E1
+  Lane A A/A session-position, and E2 operating-characteristic candidate SAPs.
+  They are not preregistered, authorize no observations, and select no powered
+  method. Their counts, thresholds, grids, and consequences remain review
+  candidates until separately frozen and registered.
 - Contract B `2.0.0-draft.1` is an immutable, repository-only normative draft
   beside v1. It binds synthetic registration, reveal, schedule, attempts,
   attrition, copied observation provenance, analysis, decision, and report
@@ -49,7 +54,7 @@ every commit is world-readable and every merge to `main` is a publication act.
   unchanged core identity, runtime dependency, and synthetic conformance pair.
   This is a repository delivery contract, not a package release or operational
   adoption; the wheel-buildable 0.2.0 package contents remain unchanged.
-- Its synthetic observation cardinality is one Aider-owner descriptor-validation
+- Its synthetic observation cardinality is one editing-client-owner descriptor-validation
   receipt and one lab-owned completion receipt per attempt; one distinct gateway
   open receipt, zero or more gateway call receipts, and one distinct gateway
   closure receipt per invocation. Zero calls are admissible only for a retained
@@ -78,9 +83,10 @@ open:
 3. **Nuisance-boundary control.** Replace the current finite `0.01` grid
    judgment with a justified continuous-boundary argument or an explicitly
    conservative bound.
-4. **Dependence.** Bound cross-task correlation induced by shared serving state,
-   session-by-arm interactions, batch composition, and prefix caching. Retain
-   session and task-pair identifiers so the assumption is auditable.
+4. **Dependence.** For Lane A, use one task pair per fresh session and attempt to
+   falsify session-position invariance rather than estimate an intra-session
+   correlation. Retain actual session position and task-pair identifiers; treat
+   shared serving state and cache effects as explicit residual threats.
 5. **Level-matched comparison.** Compare candidate procedures at the same
    demonstrated Type I level; do not attribute unmatched power gaps to one
    mechanism.

README.md

@@ -87,7 +87,8 @@ order to trust or challenge an evaluation.
   explicit decision.
 - No validated operating characteristics for the intended class-balanced design.
 - No operational registration signer, trusted timestamp path, or production owner-receipt join.
-- No validated grader study, governed 40-task suite, A/A calibration, or powered campaign.
+- No validated grader study, no public-safe validity evidence for the governed 40-task suite, no
+  A/A calibration, and no powered campaign.
 - No evidence that a candidate model or harness improves real task performance.
 
 ## Evidence and reproduction
@@ -97,6 +98,7 @@ order to trust or challenge an evaluation.
 - [Refusal example](reports/contract-v2-not-evaluable.qmd)
 - [Identity-domain specification](identity-domain.qmd)
 - [Delivery manifest](evidence/contract-v2/delivery-manifest.v1.json)
+- [Candidate calibration protocol family](docs/preregistrations/calibration-sap.md)
 - [Roadmap](PLAN.md)
 
 The repository contains two historical worked examples:
@@ -110,17 +112,17 @@ historical artifacts, not evidence for model selection.
 
 ### Verify locally
 
+Install the exact Quarto release named in `.quarto-version` (on macOS,
+`brew install --cask quarto` installs that release when Homebrew's cask matches the pin). Then run:
+
 ```bash
-python -m pip install '.[dev]'
-PYTHONPATH=src python -m unittest discover -s tests -v
-PYTHONPATH=src python -m analysis.run_method_tranche --check
-PYTHONPATH=src python -m analysis.run_contract_v2 --check
-make validate-report EVIDENCE=evidence/sample-lab-report.json
-python -m pip wheel . --no-deps -w dist/
+python -m pip install '.[dev,report]'
+bash scripts/run-merge-gate.sh
 ```
 
-Quarto is also required to render the site and HTML reports. CI runs tests, exact-result checks,
-wheel containment, report validation, and site rendering before publication.
+The script refuses a missing or mismatched Quarto version. CI reads the same version pin and invokes
+the same merge-gate script, covering Ruff, tests, exact-result checks, wheel containment, evidence
+validation, and site/report rendering before publication.
 
 ## Technical details
 

STATUS.md

@@ -16,9 +16,10 @@ ready for a powered scientific promotion decision or production use.
 
 | Item | Evidence | Status |
 |---|---|---|
-| Public site | Render and Pages workflows passed for publication commit `3ceffb1` | published |
+| Public site | Render run `29614595319` and Pages run `29614595347` passed for current published baseline `6aabf7f` | published |
 | Core version | `pyproject.toml` and frozen core identify `0.2.0`; wheel containment is tested | repository-versioned, not distributed — the version names code in this repository, not a release anyone can install from a registry |
 | Experimental calculations | Committed synthetic results regenerate byte-for-byte | decision input only — the numbers inform, but do not make, the pending method decision |
+| Calibration SAP family | Separate E4, E1, and E2 candidate protocols define reviewable objectives, counts, failure rules, and claim boundaries | not preregistered; no observations authorized; numerical choices remain candidates |
 | Contract B v2 draft | Exact schema, verifier, renderer, safety scanner, fixture, report, and delivery manifest are digest-bound (every implementation byte is hash-pinned) | synthetic conformance only — the draft evidence contract is validated against invented data, not real campaigns |
 | Independent refusal | The public verifier derives `NOT_EVALUABLE` from the synthetic fixture and rejects mutations — it recomputes the result from frozen evidence bytes and refuses any altered copy | verified for the fixture |
 | Publication scanning | Evidence inputs are recursively scanned in the parameterized report path; Contract B checks its bound public payload separately | path-scoped; no whole-repository scan — leak scanning covers the report inputs, not every file in the repository |
@@ -34,10 +35,10 @@ therefore identifies repository and wheel metadata, not a distributed release.
 | Operating characteristics | Type I error (how often the rule would be wrong), power (how often it would detect a real difference), dependence sensitivity, and feasibility are not established for the intended design |
 | Measurement validity | Grader validation, protected-task policy, and governed task selection are incomplete — the scoring instrument itself is not yet validated |
 | Operational provenance | Registration signing/timestamp verification and runtime owner receipts are not operational — runtime evidence is not yet signed and independently verified in production |
-| Calibration | No accepted grader study, A/A independence/noise-floor study, or rule-calibration study exists — nothing yet shows the measurement apparatus is quiet and consistent enough to trust |
+| Calibration | No accepted grader study, A/A session-position invariance study, or rule-calibration study exists — nothing yet shows the measurement apparatus is quiet and consistent enough to trust |
 | Real result | No powered, registered, scientifically admissible model comparison has been published |
 | Historical smoke reproducibility | Only summary statistics are public; raw per-attempt outcomes required to recompute the interval are absent — outside readers cannot independently re-derive it |
-| Publication enforcement | CI passes on current `main`, but no GitHub ruleset or branch protection currently requires those checks — nothing mechanically blocks a merge that skips them |
+| Publication enforcement | Active no-bypass ruleset `18954954` requires pull requests, signed commits, linear history, resolved review threads, and the strict `Render public evidence reports` check; approval count is zero under the disclosed solo-maintainer model | protected, but not independent review |
 
 ## Claim policy
 
@@ -55,7 +56,7 @@ recomputation, and explicit refusal behavior. They do not support claims that:
 1. Accept a scientific analysis plan and publish the corresponding versioned method contract.
 2. Complete owner-issued runtime observations and operational registration verification.
 3. Freeze the grader, task-selection record, task suite, assignment, retry, and attrition policies.
-4. Run and review grader validation, A/A independence/noise-floor calibration, and operating-
+4. Run and review grader validation, A/A session-position invariance calibration, and operating-
    characteristic analysis.
 5. Register and execute the first powered local comparison.
 6. Independently recompute, sanitize, review, and publish the result, including a no-go or
@@ -72,5 +73,5 @@ conformance vector are unchanged.
 
 ## Provenance
 
-- Status date: 2026-07-14
-- Audited public baseline: `3ceffb1474146c8b462af11112faf4c50b547ce8`
+- Status date: 2026-07-17
+- Baseline reviewed for this candidate update: `6aabf7f32e3d824ee63f2eef3a761f29be4e2d4c`

build_backend.py

@@ -7,15 +7,19 @@
 import hashlib
 import io
 import tarfile
+import tomllib
 import zipfile
 from pathlib import Path
 from typing import Any
 
-NAME = "eval-lab-methodology"
-NORMALIZED_NAME = "eval_lab_methodology"
-VERSION = "0.2.0"
-DIST_INFO = f"{NORMALIZED_NAME}-{VERSION}.dist-info"
 ROOT = Path(__file__).resolve().parent
+PROJECT = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))[
+    "project"
+]
+NAME = PROJECT["name"]
+NORMALIZED_NAME = NAME.replace("-", "_")
+VERSION = PROJECT["version"]
+DIST_INFO = f"{NORMALIZED_NAME}-{VERSION}.dist-info"
 PACKAGE_ROOT = ROOT / "src" / "eval_lab_methodology"
 
 
@@ -118,24 +122,25 @@ def _is_bytecode(path: Path) -> bool:
 
 
 def _metadata() -> str:
-    return "\n".join(
-        [
-            "Metadata-Version: 2.3",
-            f"Name: {NAME}",
-            f"Version: {VERSION}",
-            "Summary: Public statistical and reporting core for the Agentic-Coding Evaluation Lab methodology.",
-            "Author: Verlyn Nash",
-            "License-Expression: Apache-2.0",
-            "Requires-Python: >=3.11",
-            "Provides-Extra: glmm",
-            "Requires-Dist: pandas>=2.0; extra == 'glmm'",
-            "Requires-Dist: statsmodels>=0.14; extra == 'glmm'",
-            "Provides-Extra: dev",
-            "Requires-Dist: jsonschema>=4.22; extra == 'dev'",
-            "Requires-Dist: pytest>=8.0; extra == 'dev'",
-            "",
-        ]
-    )
+    lines = [
+        "Metadata-Version: 2.3",
+        f"Name: {NAME}",
+        f"Version: {VERSION}",
+        f"Summary: {PROJECT['description']}",
+        f"Author: {PROJECT['authors'][0]['name']}",
+        f"License-Expression: {PROJECT['license']}",
+        f"Requires-Python: {PROJECT['requires-python']}",
+    ]
+    for requirement in PROJECT.get("dependencies", []):
+        lines.append(f"Requires-Dist: {requirement}")
+    for extra, requirements in PROJECT.get("optional-dependencies", {}).items():
+        lines.append(f"Provides-Extra: {extra}")
+        lines.extend(
+            f"Requires-Dist: {requirement}; extra == '{extra}'"
+            for requirement in requirements
+        )
+    lines.append("")
+    return "\n".join(lines)
 
 
 def _wheel_metadata() -> str: