[codex] add deterministic v1 eval harness

docs/DECISIONS.md

@@ -448,3 +448,24 @@ the target product is home iPhone video, not GuitarSet. A silent default would
 hide a dataset-specific learned bias inside every decode. Keeping it explicit
 preserves baseline behavior while allowing the coordinator to run the exact
 home-video ablation before deciding whether to promote it.
+
+---
+
+## 2026-05-07 — Phase 8 smoke eval is deterministic without external data
+
+**Phase:** 8 (eval harness hardening)
+**Decision tree:** Phase 8 determinism gate and deferred Phase 1.5/3/4 debt audit
+**Branch taken:** **Use a dependency-light synthetic smoke scope for CI, and
+emit explicit blockers for full model-backed eval until the manifest and labels
+exist.**
+**Evidence:** `docs/EVAL_REPORTS/eval_full_20260507T000000Z.json` and
+`.md` report 0 manifest clips, all four required tiers missing, Phase 3
+preflight/fretboard labels at 0/10 and 0/5, and Phase 4 hand labels at
+0/100. `python -m scripts.eval.run --scope smoke --twice-and-diff
+--timestamp 2026-05-07T00:00:00Z` reports `deterministic=true` with a
+180-second smoke budget.
+**Reasoning:** Full eval cannot honestly produce audio-only/audio+vision/prior
+or confidence-calibration metrics without the Phase 1.5 manifest and external
+media/annotations. The smoke scope still exercises the same report writer and
+fixed output format in CI, so Phase 8 hardening can progress without masking the
+remaining data-bound acceptance debt.

docs/EVAL_REPORTS/eval_full_20260507T000000Z.json

@@ -0,0 +1,156 @@
+{
+  "ablations": [
+    {
+      "blocker": "No complete Phase 1.5 manifest plus local media/annotations are available for model-backed eval.",
+      "chord_accuracy": null,
+      "onset_f1": null,
+      "pitch_f1": null,
+      "status": "blocked",
+      "tab_f1": null,
+      "variant": "audio_only"
+    },
+    {
+      "blocker": "No complete Phase 1.5 manifest plus local media/annotations are available for model-backed eval.",
+      "chord_accuracy": null,
+      "onset_f1": null,
+      "pitch_f1": null,
+      "status": "blocked",
+      "tab_f1": null,
+      "variant": "audio_vision"
+    },
+    {
+      "blocker": "No complete Phase 1.5 manifest plus local media/annotations are available for model-backed eval.",
+      "chord_accuracy": null,
+      "onset_f1": null,
+      "pitch_f1": null,
+      "status": "blocked",
+      "tab_f1": null,
+      "variant": "audio_vision_prior"
+    }
+  ],
+  "confidence_calibration": {
+    "bins": 10,
+    "blocker": "confidence calibration is blocked until scored predictions with per-event confidence exist for manifest clips (currently 0 clips).",
+    "ece": null,
+    "metric": "ece",
+    "status": "blocked"
+  },
+  "manifest": {
+    "clip_count": 0,
+    "clip_ids": [],
+    "items": [
+      {
+        "clip_id": null,
+        "code": "MISSING_TIER",
+        "message": "Add at least one eval clip for required tier 'clean_acoustic_single_line'.",
+        "severity": "fail"
+      },
+      {
+        "clip_id": null,
+        "code": "MISSING_TIER",
+        "message": "Add at least one eval clip for required tier 'clean_acoustic_strummed'.",
+        "severity": "fail"
+      },
+      {
+        "clip_id": null,
+        "code": "MISSING_TIER",
+        "message": "Add at least one eval clip for required tier 'clean_electric'.",
+        "severity": "fail"
+      },
+      {
+        "clip_id": null,
+        "code": "MISSING_TIER",
+        "message": "Add at least one eval clip for required tier 'distorted_electric'.",
+        "severity": "fail"
+      },
+      {
+        "clip_id": null,
+        "code": "TOO_FEW_CLIPS",
+        "message": "Phase 1.5 requires >= 15 clips; found 0.",
+        "severity": "fail"
+      }
+    ],
+    "manifest_path": "data/eval/manifest.toml",
+    "missing_tiers": [
+      "clean_acoustic_single_line",
+      "clean_acoustic_strummed",
+      "clean_electric",
+      "distorted_electric"
+    ],
+    "passed": false,
+    "present_tiers": []
+  },
+  "phase_debt": {
+    "phase_1_5": {
+      "command": "tabvision-eval --manifest tabvision/data/eval/manifest.toml --check",
+      "gate": "manifest completeness and all required tiers represented"
+    },
+    "phase_3": {
+      "fretboard": {
+        "command": "pytest -m fretboard_eval tests/eval/test_phase3_eval.py",
+        "required_labels": 5,
+        "status": "blocked",
+        "usable_labels": 0
+      },
+      "guitar_detector": {
+        "evidence": "docs/DECISIONS.md#2026-05-05-phase-3-detector-acceptance",
+        "metric": "neck mAP50=0.995",
+        "status": "passed_documented"
+      },
+      "preflight": {
+        "command": "pytest -m preflight_eval tests/eval/test_phase3_eval.py",
+        "required_labels": 10,
+        "status": "blocked",
+        "usable_labels": 0
+      }
+    },
+    "phase_4": {
+      "hand": {
+        "command": "pytest -m hand_eval tests/eval/test_phase4_eval.py",
+        "required_fretting_labels": 100,
+        "status": "blocked",
+        "usable_fretting_labels": 0
+      }
+    }
+  },
+  "schema_version": 1,
+  "scope": "full",
+  "seed": 0,
+  "smoke": {
+    "enabled": false,
+    "passed": false,
+    "subset": []
+  },
+  "smoke_budget_s": 180.0,
+  "tier_breakdown": [
+    {
+      "clip_count": 0,
+      "status": "blocked",
+      "tab_f1": null,
+      "tab_f1_target": 0.94,
+      "tier": "clean_acoustic_single_line"
+    },
+    {
+      "clip_count": 0,
+      "status": "blocked",
+      "tab_f1": null,
+      "tab_f1_target": 0.86,
+      "tier": "clean_acoustic_strummed"
+    },
+    {
+      "clip_count": 0,
+      "status": "blocked",
+      "tab_f1": null,
+      "tab_f1_target": 0.9,
+      "tier": "clean_electric"
+    },
+    {
+      "clip_count": 0,
+      "status": "blocked",
+      "tab_f1": null,
+      "tab_f1_target": 0.82,
+      "tier": "distorted_electric"
+    }
+  ],
+  "timestamp": "2026-05-07T00:00:00Z"
+}

docs/EVAL_REPORTS/eval_full_20260507T000000Z.md

@@ -0,0 +1,44 @@
+# Eval Debt And Harness Report (full)
+
+Timestamp: `2026-05-07T00:00:00Z`
+Seed: `0`
+Smoke budget target: < 180 s
+
+## Phase 1.5 Manifest
+
+- Passed: `False`
+- Clips: `0`
+- Missing tiers: `clean_acoustic_single_line, clean_acoustic_strummed, clean_electric, distorted_electric`
+
+## Per-Tier Breakdown
+
+| Tier | Clips | Target Tab F1 | Status | Current Tab F1 |
+|---|---:|---:|---|---:|
+| clean_acoustic_single_line | 0 | 0.94 | blocked |  |
+| clean_acoustic_strummed | 0 | 0.86 | blocked |  |
+| clean_electric | 0 | 0.90 | blocked |  |
+| distorted_electric | 0 | 0.82 | blocked |  |
+
+## Phase 3/4 Acceptance Debt
+
+| Gate | Status | Evidence / Blocker | Command |
+|---|---|---|---|
+| Phase 3 guitar detector | passed_documented | neck mAP50=0.995 (docs/DECISIONS.md#2026-05-05-phase-3-detector-acceptance) | current report |
+| Phase 3 preflight | blocked | 0/10 labels | `pytest -m preflight_eval tests/eval/test_phase3_eval.py` |
+| Phase 3 fretboard | blocked | 0/5 labels | `pytest -m fretboard_eval tests/eval/test_phase3_eval.py` |
+| Phase 4 hand | blocked | 0/100 fretting labels | `pytest -m hand_eval tests/eval/test_phase4_eval.py` |
+
+## Ablations
+
+| Variant | Status | Onset F1 | Pitch F1 | Tab F1 | Chord Acc | Blocker |
+|---|---|---:|---:|---:|---:|---|
+| audio_only | blocked |  |  |  |  | No complete Phase 1.5 manifest plus local media/annotations are available for model-backed eval. |
+| audio_vision | blocked |  |  |  |  | No complete Phase 1.5 manifest plus local media/annotations are available for model-backed eval. |
+| audio_vision_prior | blocked |  |  |  |  | No complete Phase 1.5 manifest plus local media/annotations are available for model-backed eval. |
+
+## Confidence Calibration
+
+- Status: `blocked`
+- Metric: `ece` with `10` bins
+- ECE: ``
+- Blocker: confidence calibration is blocked until scored predictions with per-event confidence exist for manifest clips (currently 0 clips).

tabvision/data/eval/manifest.toml

@@ -0,0 +1,17 @@
+# Phase 1.5 eval manifest.
+#
+# This file is intentionally a checked-in placeholder until the external
+# datasets are registered under $TABVISION_DATA_ROOT. The validator reports
+# the concrete missing clips and tiers:
+#
+#   tabvision-eval --manifest tabvision/data/eval/manifest.toml --check
+#
+# Required clip schema:
+#
+# [[clips]]
+# id = "guitarset-05-example"
+# tier = "clean_acoustic_single_line"
+# source = "GuitarSet"
+# split = "validation"
+# media_path = "$TABVISION_DATA_ROOT/guitarset/audio_mono-mic/05_example_mic.wav"
+# annotation_path = "$TABVISION_DATA_ROOT/guitarset/annotation/05_example.jams"

tabvision/pyproject.toml

@@ -80,6 +80,7 @@ dev = [
 
 [project.scripts]
 tabvision = "tabvision.cli:main"
+tabvision-eval = "tabvision.eval.runner:main"
 
 [tool.hatch.version]
 path = "tabvision/__init__.py"

tabvision/scripts/eval/run.py

@@ -0,0 +1,13 @@
+"""One-command eval runner.
+
+Examples:
+    python -m scripts.eval.run --scope full
+    python -m scripts.eval.run --scope smoke --twice-and-diff
+"""
+
+from __future__ import annotations
+
+from tabvision.eval.runner import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())