From 7f18579dfb46458d525700b13d9cd1ddb453f20e Mon Sep 17 00:00:00 2001 From: Graphanov Date: Wed, 3 Jun 2026 16:25:56 +0200 Subject: [PATCH] Preserve v3 claim discipline after private pilot exposed telemetry gaps Constraint: Private Gate 6 pilot ended no-support; visual/product ranking stayed blocked. Rejected: Treat Lane B token/evidence volume as score | It would rig benchmark and hide final mechanical tie. Confidence: high Scope-risk: moderate Directive: Keep token/cost telemetry separate from mechanical scoring unless a future frozen protocol defines an efficiency track. Tested: git diff --check; python3 scripts/validate_v3_schemas.py; python3 scripts/run_v3_mechanical_smokes.py; python3 scripts/run_v3_feedback_parity_smoke.py; python3 scripts/run_v3_visual_smokes.py --visual-out v3/fixtures/visual/out; python3 scripts/validate_v3_visual_packages.py v3/fixtures/visual/out/visual-package.json; python3 scripts/validate_v3_workflow_scenarios.py; python3 scripts/validate_v3_campaign_protocol.py; python3 scripts/scan_changed_public_safety.py; python3 scripts/run_v3_token_telemetry_smoke.py; python3 scripts/check_v3_feedback_parity.py ; cargo fmt --all --check; cargo build --workspace --quiet; cargo clippy --workspace -- -D warnings; cargo test --workspace --quiet Not-tested: Applying --write-records to historical private run root; public visual reranking of private selected packages. Co-authored-by: OmX --- scripts/collect_v3_token_telemetry.py | 332 ++++++++++++++++++ scripts/render_v3_private_report.py | 269 ++++++++++++++ scripts/run_v3_token_telemetry_smoke.py | 120 +++++++ scripts/run_v3_visual_smokes.py | 8 +- scripts/validate_v3_schemas.py | 12 +- scripts/validate_v3_visual_packages.py | 13 +- v3/EVIDENCE_AND_CLAIMS.md | 20 +- v3/IMPLEMENTATION_PLAN.md | 2 + v3/MANIFEST_AND_CAPTURE.md | 7 + v3/SCORING_MODEL.md | 38 ++ v3/VISUAL_RUBRIC.md | 3 +- v3/examples/valid/run-record.foundation.json | 25 ++ .../valid/visual-package.foundation.json | 2 + v3/fixtures/visual/valid-artifact/capture.py | 7 +- v3/run-record.schema.json | 130 +++++++ v3/visual-package.schema.json | 10 + 16 files changed, 981 insertions(+), 17 deletions(-) create mode 100755 scripts/collect_v3_token_telemetry.py create mode 100755 scripts/render_v3_private_report.py create mode 100755 scripts/run_v3_token_telemetry_smoke.py diff --git a/scripts/collect_v3_token_telemetry.py b/scripts/collect_v3_token_telemetry.py new file mode 100755 index 0000000..d08d3be --- /dev/null +++ b/scripts/collect_v3_token_telemetry.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +"""Collect v3 private-campaign token telemetry from Codex stderr logs. + +This is runner-layout aware and scorer-neutral. It records runtime/cost +telemetry separately from mechanical, visual, workflow, and evidence tracks. +""" +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any + +TOKEN_MARKER_RE = re.compile(r"^\s*tokens used\s*$", re.IGNORECASE) +TOKEN_VALUE_RE = re.compile(r"^\s*([0-9][0-9,]*)\s*$") +TOKEN_INLINE_RE = re.compile(r"\b([0-9][0-9,]*)\s+tokens used\b", re.IGNORECASE) +GENERATION_RE = re.compile(r"records/pilot-seed-(\d+)/lane-([^/]+)/generation-(\d+)$") + + +def load_json(path: Path) -> Any: + return json.loads(path.read_text()) + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def require(condition: bool, message: str) -> None: + if not condition: + raise SystemExit(message) + + +def rel(path: Path, root: Path) -> str: + try: + return path.relative_to(root).as_posix() + except ValueError: + return str(path) + + +def parse_codex_tokens_used(stderr_path: Path) -> int | None: + if not stderr_path.exists(): + return None + lines = stderr_path.read_text(errors="replace").splitlines() + values: list[int] = [] + for index, line in enumerate(lines): + for match in TOKEN_INLINE_RE.finditer(line): + values.append(int(match.group(1).replace(",", ""))) + if TOKEN_MARKER_RE.match(line): + for next_line in lines[index + 1:index + 5]: + value_match = TOKEN_VALUE_RE.match(next_line) + if value_match: + values.append(int(value_match.group(1).replace(",", ""))) + break + return values[-1] if values else None + + +def generation_dirs(run_root: Path) -> list[Path]: + records = run_root / "records" + return sorted(path for path in records.glob("pilot-seed-*/lane-*/generation-*") if path.is_dir()) + + +def generation_identity(path: Path, run_root: Path) -> tuple[int, str, int]: + relative = rel(path, run_root) + match = GENERATION_RE.fullmatch(relative) + require(match is not None, f"unexpected generation path: {relative}") + assert match is not None + return int(match.group(1)), match.group(2).upper(), int(match.group(3)) + + +def mechanical_status(path: Path, run_root: Path) -> dict[str, Any]: + for filename in ("run-record.json", "v3-result.json"): + candidate = path / filename + if candidate.exists(): + data = load_json(candidate) + mechanical = data.get("mechanical", {}) if isinstance(data, dict) else {} + if isinstance(mechanical, dict): + return { + "ranked": mechanical.get("ranked"), + "passCount": mechanical.get("passCount"), + "totalAcs": mechanical.get("totalAcs"), + "failedAcs": mechanical.get("failedAcs", []), + "sourceRef": rel(candidate, run_root), + } + return {"ranked": None, "passCount": None, "totalAcs": None, "failedAcs": [], "sourceRef": ""} + + +def runtime_telemetry(stderr_ref: str, total_tokens: int | None) -> dict[str, Any]: + unavailable = ["input", "output", "cachedInput", "reasoning"] + if total_tokens is None: + unavailable = ["total", *unavailable] + return { + "source": { + "kind": "codex-stderr", + "ref": stderr_ref, + "parser": "codex-cli-tokens-used-line-v1", + }, + "tokens": { + "total": total_tokens, + "input": None, + "output": None, + "cachedInput": None, + "reasoning": None, + "unavailableFields": unavailable, + }, + "cost": { + "estimatedUsd": None, + "currency": None, + "unavailableReason": "Codex stderr exposes a reliable total token count only; no versioned pricing table or token split was applied.", + }, + } + + +def update_run_record(generation_dir: Path, telemetry: dict[str, Any]) -> None: + path = generation_dir / "run-record.json" + if not path.exists(): + return + data = load_json(path) + require(isinstance(data, dict), f"run-record must be a JSON object: {path}") + data["runtimeTelemetry"] = telemetry + write_json(path, data) + + +def add_nested_total(target: dict[str, Any], seed: int, lane: str, tokens: int | None) -> None: + seed_key = str(seed) + target.setdefault(seed_key, {}) + current = target[seed_key].get(lane) + if tokens is None: + target[seed_key][lane] = current + else: + target[seed_key][lane] = (current or 0) + tokens + + +def percent_delta(delta: int | None, baseline: int | None) -> float | None: + if delta is None or not baseline: + return None + return round((delta / baseline) * 100, 6) + + +def delta_summary(by_lane: dict[str, int]) -> dict[str, Any]: + a = by_lane.get("A") + b = by_lane.get("B") + delta = b - a if a is not None and b is not None else None + return { + "laneA": a, + "laneB": b, + "laneBMinusLaneA": delta, + "laneBPercentDeltaVsLaneA": percent_delta(delta, a), + } + + +def aggregate(entries: list[dict[str, Any]]) -> dict[str, Any]: + by_seed_lane: dict[str, Any] = {} + by_lane: dict[str, int] = {} + missing_token_entries = 0 + for entry in entries: + tokens = entry["tokens"]["total"] + if tokens is None: + missing_token_entries += 1 + continue + add_nested_total(by_seed_lane, entry["taskSeed"], entry["laneId"], tokens) + by_lane[entry["laneId"]] = by_lane.get(entry["laneId"], 0) + tokens + + first_ceiling_by_seed_lane: dict[str, Any] = {} + by_pair: dict[tuple[int, str], list[dict[str, Any]]] = {} + for entry in entries: + by_pair.setdefault((entry["taskSeed"], entry["laneId"]), []).append(entry) + for (seed, lane), rows in sorted(by_pair.items()): + running = 0 + found: dict[str, Any] | None = None + for row in sorted(rows, key=lambda item: item["generation"]): + tokens = row["tokens"]["total"] + if tokens is not None: + running += tokens + mechanical = row["mechanical"] + if mechanical["ranked"] is True and mechanical["passCount"] == mechanical["totalAcs"] and mechanical["totalAcs"]: + found = { + "generation": row["generation"], + "tokens": running, + "passCount": mechanical["passCount"], + "totalAcs": mechanical["totalAcs"], + } + break + first_ceiling_by_seed_lane.setdefault(str(seed), {})[lane] = found + + first_ceiling_by_lane: dict[str, int] = {} + for lanes in first_ceiling_by_seed_lane.values(): + for lane, value in lanes.items(): + if isinstance(value, dict) and isinstance(value.get("tokens"), int): + first_ceiling_by_lane[lane] = first_ceiling_by_lane.get(lane, 0) + value["tokens"] + + improvement_rows: list[dict[str, Any]] = [] + for (seed, lane), rows in sorted(by_pair.items()): + previous_pass: int | None = None + for row in sorted(rows, key=lambda item: item["generation"]): + current_pass = row["mechanical"]["passCount"] + tokens = row["tokens"]["total"] + if previous_pass is None or not isinstance(current_pass, int): + improvement_rows.append({ + "taskSeed": seed, + "laneId": lane, + "generation": row["generation"], + "deltaPassCount": None, + "tokens": tokens, + "tokensPerPassedAcImprovement": None, + "meaningful": False, + "reason": "no previous generation baseline", + }) + else: + delta = current_pass - previous_pass + improvement_rows.append({ + "taskSeed": seed, + "laneId": lane, + "generation": row["generation"], + "deltaPassCount": delta, + "tokens": tokens, + "tokensPerPassedAcImprovement": round(tokens / delta, 6) if isinstance(tokens, int) and delta > 0 else None, + "meaningful": delta > 0 and isinstance(tokens, int), + "reason": "pass-count improved" if delta > 0 else "no pass-count improvement", + }) + previous_pass = current_pass if isinstance(current_pass, int) else previous_pass + + return { + "tokensBySeedLane": by_seed_lane, + "tokensByLane": by_lane, + "laneDeltaFullRun": delta_summary(by_lane), + "tokensUntilFirstRankedMechanicalCeiling": { + "bySeedLane": first_ceiling_by_seed_lane, + "byLane": first_ceiling_by_lane, + "laneDelta": delta_summary(first_ceiling_by_lane), + }, + "tokensPerPassedAcImprovement": improvement_rows, + "missingTokenEntryCount": missing_token_entries, + } + + +def collect(run_root: Path, *, write_records: bool) -> dict[str, Any]: + entries: list[dict[str, Any]] = [] + for gen_dir in generation_dirs(run_root): + seed, lane, generation = generation_identity(gen_dir, run_root) + stderr_path = gen_dir / "codex-stderr.log" + stderr_ref = rel(stderr_path, run_root) + tokens = parse_codex_tokens_used(stderr_path) + telemetry = runtime_telemetry(stderr_ref, tokens) + mechanical = mechanical_status(gen_dir, run_root) + token_record = { + "schemaVersion": "2000m.v3.runtime-telemetry.v1", + "taskSeed": seed, + "laneId": lane, + "generation": generation, + "runtimeTelemetry": telemetry, + } + if write_records: + write_json(gen_dir / "token-telemetry.json", token_record) + update_run_record(gen_dir, telemetry) + entries.append({ + "taskSeed": seed, + "laneId": lane, + "generation": generation, + "generationRef": rel(gen_dir, run_root), + "stderrRef": stderr_ref, + "tokens": telemetry["tokens"], + "cost": telemetry["cost"], + "mechanical": mechanical, + }) + return { + "schemaVersion": "2000m.v3.token-telemetry-summary.v1", + "runRoot": str(run_root), + "parser": "codex-cli-tokens-used-line-v1", + "trackBoundary": "runtime token/cost telemetry is separate from mechanical correctness and is not a mechanical score component", + "entries": entries, + "aggregates": aggregate(entries), + } + + +def render_markdown(summary: dict[str, Any]) -> str: + full = summary["aggregates"]["laneDeltaFullRun"] + first = summary["aggregates"]["tokensUntilFirstRankedMechanicalCeiling"]["laneDelta"] + lines = [ + "# v3 token telemetry summary", + "", + "Token/cost telemetry is separate from mechanical correctness. It is not a score component.", + "", + "## Lane totals", + "", + f"- Full run Lane A: {full['laneA']}", + f"- Full run Lane B: {full['laneB']}", + f"- Lane B minus Lane A: {full['laneBMinusLaneA']} ({full['laneBPercentDeltaVsLaneA']}%)", + "", + "## Until first ranked mechanical ceiling", + "", + f"- Lane A: {first['laneA']}", + f"- Lane B: {first['laneB']}", + f"- Lane B minus Lane A: {first['laneBMinusLaneA']} ({first['laneBPercentDeltaVsLaneA']}%)", + "", + "## Unavailable fields", + "", + "- Codex stderr exposed total tokens only.", + "- Input/output/cached/reasoning token splits are null.", + "- Estimated cost is null because no versioned pricing table was applied.", + "", + ] + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Collect v3 private campaign token telemetry") + parser.add_argument("run_root", type=Path) + parser.add_argument("--write-records", action="store_true", help="write token-telemetry.json files and update run-record.json runtimeTelemetry fields") + parser.add_argument("--json-out", type=Path, help="write aggregate telemetry JSON") + parser.add_argument("--markdown-out", type=Path, help="write aggregate telemetry markdown") + args = parser.parse_args() + + run_root = args.run_root.resolve() + require(run_root.exists() and run_root.is_dir(), f"run root does not exist: {run_root}") + summary = collect(run_root, write_records=args.write_records) + if args.json_out: + write_json(args.json_out if args.json_out.is_absolute() else Path.cwd() / args.json_out, summary) + if args.markdown_out: + out = args.markdown_out if args.markdown_out.is_absolute() else Path.cwd() / args.markdown_out + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(render_markdown(summary)) + if not args.json_out and not args.markdown_out: + json.dump(summary, sys.stdout, indent=2, sort_keys=True) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/render_v3_private_report.py b/scripts/render_v3_private_report.py new file mode 100755 index 0000000..0ed08ab --- /dev/null +++ b/scripts/render_v3_private_report.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""Render a conservative v3 private campaign report from run records.""" +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Any + +from collect_v3_token_telemetry import collect as collect_token_telemetry + +GENERATION_RE = re.compile(r"records/pilot-seed-(\d+)/lane-([^/]+)/generation-(\d+)$") + + +def load_json(path: Path) -> Any: + return json.loads(path.read_text()) + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def rel(path: Path, root: Path) -> str: + try: + return path.relative_to(root).as_posix() + except ValueError: + return str(path) + + +def generation_dirs(run_root: Path) -> list[Path]: + return sorted(path for path in (run_root / "records").glob("pilot-seed-*/lane-*/generation-*") if path.is_dir()) + + +def generation_identity(path: Path, run_root: Path) -> tuple[int, str, int]: + match = GENERATION_RE.fullmatch(rel(path, run_root)) + if not match: + raise SystemExit(f"unexpected generation path: {path}") + return int(match.group(1)), match.group(2).upper(), int(match.group(3)) + + +def read_generation(path: Path, run_root: Path) -> dict[str, Any]: + seed, lane, generation = generation_identity(path, run_root) + result = load_json(path / "v3-result.json") + run_record = load_json(path / "run-record.json") + visual_status_path = path / "visual-status.json" + visual_status = load_json(visual_status_path) if visual_status_path.exists() else result.get("visual", {}) + return { + "taskSeed": seed, + "laneId": lane, + "generation": generation, + "generationRef": rel(path, run_root), + "resultRef": rel(path / "v3-result.json", run_root), + "runRecordRef": rel(path / "run-record.json", run_root), + "mechanical": result["mechanical"], + "visual": visual_status, + "workflow": run_record.get("workflow", {}), + } + + +def selected_generations(entries: list[dict[str, Any]]) -> dict[str, dict[str, dict[str, Any]]]: + grouped: dict[tuple[int, str], list[dict[str, Any]]] = {} + for entry in entries: + grouped.setdefault((entry["taskSeed"], entry["laneId"]), []).append(entry) + selected: dict[str, dict[str, dict[str, Any]]] = {} + for (seed, lane), rows in sorted(grouped.items()): + def key(row: dict[str, Any]) -> tuple[int, float, int]: + mechanical = row["mechanical"] + return ( + int(mechanical.get("passCount") or 0), + float(mechanical.get("compositeScore") or 0), + int(row["generation"]), + ) + best = max(rows, key=key) + selected.setdefault(str(seed), {})[lane] = best + return selected + + +def trajectory(entries: list[dict[str, Any]]) -> dict[str, dict[str, str]]: + grouped: dict[tuple[int, str], list[dict[str, Any]]] = {} + for entry in entries: + grouped.setdefault((entry["taskSeed"], entry["laneId"]), []).append(entry) + out: dict[str, dict[str, str]] = {} + for (seed, lane), rows in sorted(grouped.items()): + parts = [] + for row in sorted(rows, key=lambda item: item["generation"]): + mechanical = row["mechanical"] + parts.append(f"{mechanical.get('passCount')}/{mechanical.get('totalAcs')}") + out.setdefault(str(seed), {})[lane] = " -> ".join(parts) + return out + + +def selected_summary(selected: dict[str, dict[str, dict[str, Any]]]) -> dict[str, Any]: + out: dict[str, Any] = {} + for seed, lanes in selected.items(): + out[seed] = {} + for lane, row in lanes.items(): + mechanical = row["mechanical"] + visual = row["visual"] + workflow = row["workflow"] + out[seed][lane] = { + "generation": row["generation"], + "passCount": mechanical.get("passCount"), + "totalAcs": mechanical.get("totalAcs"), + "ranked": mechanical.get("ranked"), + "resultRef": row["resultRef"], + "visualRanked": visual.get("ranked"), + "visualBlockReason": visual.get("blockReason"), + "workflow": workflow, + } + return out + + +def feedback_parity(run_root: Path) -> dict[str, Any]: + reports = sorted((run_root / "feedback-parity").glob("generation-*.json")) + if not reports: + return {"pass": None, "latestReportRef": None} + latest = reports[-1] + data = load_json(latest) + return {"pass": data.get("pass"), "latestReportRef": rel(latest, run_root)} + + +def visual_blocked(selected: dict[str, dict[str, dict[str, Any]]]) -> bool: + for lanes in selected.values(): + for row in lanes.values(): + if row["visual"].get("ranked") is not True: + return True + return False + + +def final_mechanical_tie(selected: dict[str, dict[str, dict[str, Any]]]) -> bool: + for lanes in selected.values(): + if not {"A", "B"}.issubset(lanes): + return False + a = lanes["A"]["mechanical"] + b = lanes["B"]["mechanical"] + if a.get("passCount") != b.get("passCount") or a.get("totalAcs") != b.get("totalAcs"): + return False + return True + + +def verdict_rationale(selected: dict[str, dict[str, dict[str, Any]]], token_summary: dict[str, Any], parity: dict[str, Any]) -> list[str]: + reasons: list[str] = [] + if final_mechanical_tie(selected): + reasons.append("Final selected mechanical outcomes tied across Lane A and Lane B.") + else: + reasons.append("Final selected mechanical outcomes did not establish a predeclared Lane B advantage.") + if visual_blocked(selected): + reasons.append("At least one selected visual/product record is rank-blocked, so no visual/product claim is valid.") + reasons.append("Workflow status is reported separately and is not independent output-quality proof.") + if parity.get("pass") is True: + reasons.append("Feedback parity passed for the latest guard report.") + else: + reasons.append("Feedback parity is missing or failed, which blocks support claims.") + delta = token_summary["aggregates"]["laneDeltaFullRun"] + if delta.get("laneBMinusLaneA") is not None: + reasons.append( + f"Lane B used {delta['laneBMinusLaneA']} more full-run tokens than Lane A ({delta['laneBPercentDeltaVsLaneA']}%)." + ) + return reasons + + +def render(run_root: Path) -> dict[str, Any]: + entries = [read_generation(path, run_root) for path in generation_dirs(run_root)] + selected = selected_generations(entries) + token_summary = collect_token_telemetry(run_root, write_records=False) + parity = feedback_parity(run_root) + return { + "schemaVersion": "2000m.v3.private-final-report.v2", + "runRoot": str(run_root), + "claimBoundary": "private-evidence-only", + "finalPrivateVerdict": "no support", + "trajectorySummary": trajectory(entries), + "selectedGenerations": selected_summary(selected), + "visualProductTrack": { + "blocked": visual_blocked(selected), + "note": "Native capture validity is a prerequisite for visual/product ranking.", + }, + "workflowTrack": { + "note": "Workflow status is reported separately and is not a mechanical score component or output-quality proof.", + }, + "feedbackParity": parity, + "tokenTelemetry": { + "trackBoundary": token_summary["trackBoundary"], + "aggregates": token_summary["aggregates"], + }, + "verdictRationale": verdict_rationale(selected, token_summary, parity), + "ownerGates": [ + "commit", + "push", + "pull-request", + "merge", + "release", + "publish", + "new public claim", + ], + } + + +def render_markdown(report: dict[str, Any]) -> str: + token_full = report["tokenTelemetry"]["aggregates"]["laneDeltaFullRun"] + token_first = report["tokenTelemetry"]["aggregates"]["tokensUntilFirstRankedMechanicalCeiling"]["laneDelta"] + lines = [ + "# 2000m v3 private report", + "", + f"Verdict: {report['finalPrivateVerdict']}", + f"Claim boundary: {report['claimBoundary']}", + "", + "## Mechanical trajectory", + "", + ] + for seed, lanes in report["trajectorySummary"].items(): + for lane, text in lanes.items(): + selected = report["selectedGenerations"][seed][lane] + lines.append( + f"- seed {seed} lane {lane}: {text}; selected gen {selected['generation']} = {selected['passCount']}/{selected['totalAcs']} ranked={selected['ranked']}" + ) + lines.extend([ + "", + "## Visual/product", + "", + f"- blocked: {report['visualProductTrack']['blocked']}", + f"- note: {report['visualProductTrack']['note']}", + "", + "## Token/cost telemetry", + "", + f"- full-run Lane A tokens: {token_full['laneA']}", + f"- full-run Lane B tokens: {token_full['laneB']}", + f"- Lane B minus Lane A: {token_full['laneBMinusLaneA']} ({token_full['laneBPercentDeltaVsLaneA']}%)", + f"- until first mechanical ceiling Lane A tokens: {token_first['laneA']}", + f"- until first mechanical ceiling Lane B tokens: {token_first['laneB']}", + f"- until-first-ceiling Lane B minus Lane A: {token_first['laneBMinusLaneA']} ({token_first['laneBPercentDeltaVsLaneA']}%)", + "", + "## Feedback parity", + "", + f"- pass: {report['feedbackParity']['pass']}", + f"- latest report: {report['feedbackParity']['latestReportRef']}", + "", + "## Verdict rationale", + "", + *[f"- {reason}" for reason in report["verdictRationale"]], + "", + ]) + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Render v3 private campaign report") + parser.add_argument("run_root", type=Path) + parser.add_argument("--json-out", type=Path) + parser.add_argument("--markdown-out", type=Path) + args = parser.parse_args() + + run_root = args.run_root.resolve() + report = render(run_root) + if args.json_out: + write_json(args.json_out if args.json_out.is_absolute() else Path.cwd() / args.json_out, report) + if args.markdown_out: + out = args.markdown_out if args.markdown_out.is_absolute() else Path.cwd() / args.markdown_out + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(render_markdown(report)) + if not args.json_out and not args.markdown_out: + print(render_markdown(report)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_v3_token_telemetry_smoke.py b/scripts/run_v3_token_telemetry_smoke.py new file mode 100755 index 0000000..cd9136b --- /dev/null +++ b/scripts/run_v3_token_telemetry_smoke.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Run public-safe smoke coverage for v3 token telemetry collection.""" +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] + + +def load_json(path: Path) -> Any: + return json.loads(path.read_text()) + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def run(cmd: list[str], cwd: Path = ROOT) -> None: + print("$", " ".join(cmd)) + subprocess.run(cmd, cwd=cwd, check=True) + + +def run_record(seed: int, lane: str, generation: int, pass_count: int, ranked: bool) -> dict[str, Any]: + data = load_json(ROOT / "v3" / "examples" / "valid" / "run-record.foundation.json") + data["taskSeed"] = seed + data["laneId"] = lane + data["mechanical"]["ranked"] = ranked + data["mechanical"]["passCount"] = pass_count + data["mechanical"]["totalAcs"] = 24 + data["mechanical"]["failedAcs"] = [] if pass_count == 24 else ["M16"] + data["mechanical"]["resultJsonRef"] = ( + f"records/pilot-seed-{seed}/lane-{lane}/generation-{generation:02d}/v3-result.json" + ) + data["finalRecommendation"]["decision"] = "stop" if ranked else "continue" + return data + + +def result_record(seed: int, lane: str, generation: int, pass_count: int, ranked: bool) -> dict[str, Any]: + data = load_json(ROOT / "v3" / "examples" / "valid" / "result.foundation.json") + data["taskSeed"] = seed + data["laneId"] = lane + data["mechanical"]["ranked"] = ranked + data["mechanical"]["passCount"] = pass_count + data["mechanical"]["totalAcs"] = 24 + data["mechanical"]["failedAcs"] = [] if pass_count == 24 else ["M16"] + data["mechanical"]["compositeScore"] = 100 if pass_count == 24 else pass_count / 24 * 100 + data["mechanical"]["resultJsonRef"] = ( + f"records/pilot-seed-{seed}/lane-{lane}/generation-{generation:02d}/v3-result.json" + ) + data["workflow"]["finalRecommendation"] = "stop" if ranked else "continue" + return data + + +def write_generation(run_root: Path, seed: int, lane: str, generation: int, pass_count: int, ranked: bool, tokens: int) -> Path: + gen_dir = run_root / "records" / f"pilot-seed-{seed}" / f"lane-{lane}" / f"generation-{generation:02d}" + gen_dir.mkdir(parents=True, exist_ok=True) + write_json(gen_dir / "run-record.json", run_record(seed, lane, generation, pass_count, ranked)) + write_json(gen_dir / "v3-result.json", result_record(seed, lane, generation, pass_count, ranked)) + (gen_dir / "codex-stderr.log").write_text(f"synthetic output\n\ntokens used\n{tokens:,}\n") + return gen_dir + + +def main() -> int: + with tempfile.TemporaryDirectory(prefix="m2000-v3-token-telemetry-") as temp: + run_root = Path(temp) + write_generation(run_root, 101, "A", 1, 22, False, 1000) + write_generation(run_root, 101, "A", 2, 24, True, 2000) + write_generation(run_root, 101, "B", 1, 18, False, 3000) + write_generation(run_root, 101, "B", 2, 24, True, 4000) + summary_path = run_root / "token-telemetry-summary.json" + markdown_path = run_root / "token-telemetry-summary.md" + report_path = run_root / "final-private-report.json" + run([ + sys.executable, + "scripts/collect_v3_token_telemetry.py", + str(run_root), + "--write-records", + "--json-out", + str(summary_path), + "--markdown-out", + str(markdown_path), + ]) + run([ + sys.executable, + "scripts/render_v3_private_report.py", + str(run_root), + "--json-out", + str(report_path), + ]) + summary = load_json(summary_path) + report = load_json(report_path) + assert summary["aggregates"]["tokensByLane"] == {"A": 3000, "B": 7000}, summary + assert summary["aggregates"]["laneDeltaFullRun"]["laneBMinusLaneA"] == 4000, summary + assert summary["aggregates"]["tokensUntilFirstRankedMechanicalCeiling"]["byLane"] == {"A": 3000, "B": 7000}, summary + assert report["finalPrivateVerdict"] == "no support", report + assert report["visualProductTrack"]["blocked"] is True, report + assert report["tokenTelemetry"]["aggregates"]["laneDeltaFullRun"]["laneBMinusLaneA"] == 4000, report + meaningful = [ + row for row in summary["aggregates"]["tokensPerPassedAcImprovement"] + if row["meaningful"] + ] + assert len(meaningful) == 2, meaningful + for path in sorted(run_root.glob("records/pilot-seed-*/lane-*/generation-*/run-record.json")): + run([sys.executable, "scripts/validate_v3_schemas.py", str(path)]) + record = load_json(path) + assert record["runtimeTelemetry"]["tokens"]["input"] is None, record + assert "input" in record["runtimeTelemetry"]["tokens"]["unavailableFields"], record + assert "Token/cost telemetry is separate from mechanical correctness" in markdown_path.read_text() + print("OK: v3 token telemetry smoke parsed totals, wrote run-record telemetry, and validated schemas") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_v3_visual_smokes.py b/scripts/run_v3_visual_smokes.py index 9e07aa9..8d41d3c 100755 --- a/scripts/run_v3_visual_smokes.py +++ b/scripts/run_v3_visual_smokes.py @@ -27,10 +27,6 @@ def sha256_file(path: Path) -> str: return sha256_bytes(path.read_bytes()) -def sha256_pair(first: Path, second: Path) -> str: - return sha256_bytes(first.read_bytes() + second.read_bytes()) - - def load_json(path: Path) -> dict[str, Any]: return json.loads(path.read_text()) @@ -163,14 +159,16 @@ def main() -> int: "seed": seed, "captureCommand": "python3 capture.py --seed {seed} --window {window} --out {out}", "screenshotRef": screenshot_ref, + "screenshotChecksum": sha256_file(out / screenshot_ref), "replayRef": replay_ref, + "replayChecksum": sha256_file(out / replay_ref), "frameMetadataRef": frame_ref, "rubricMetadataRef": rubric_ref, "fps": frames["fps"], "frameCount": frames["frameCount"], "inputSequenceRef": "visual-package/inputs/empty-inputs.json", "stateChecksum": frames["stateChecksum"], - "frameChecksum": sha256_pair(out / screenshot_ref, out / replay_ref), + "frameChecksum": frames["frameChecksum"], "warnings": [] }) labels[label] = {"seed": seed, "window": window, "sealedLaneKey": "artifact-01"} diff --git a/scripts/validate_v3_schemas.py b/scripts/validate_v3_schemas.py index 1dac631..3182dea 100755 --- a/scripts/validate_v3_schemas.py +++ b/scripts/validate_v3_schemas.py @@ -286,6 +286,16 @@ def validate_semantics(data: dict[str, Any], schema_version: str) -> None: require(visual["blockReason"] == "none", "ranked run-record visual status must use blockReason none") require(bool(visual["visualPackageRef"].strip()), "ranked run-record visual status requires non-empty visualPackageRef") require(bool(visual.get("captureCommandResultRef", "").strip()), "ranked run-record visual status requires non-empty captureCommandResultRef") + telemetry = data.get("runtimeTelemetry") + if telemetry is not None: + tokens = telemetry["tokens"] + unavailable = set(tokens["unavailableFields"]) + for key in ("total", "input", "output", "cachedInput", "reasoning"): + if tokens[key] is None: + require(key in unavailable, f"runtimeTelemetry.tokens.{key} null must be listed in unavailableFields") + cost = telemetry["cost"] + if cost["estimatedUsd"] is None: + require(bool(cost["unavailableReason"].strip()), "runtimeTelemetry cost unavailableReason is required when estimatedUsd is null") elif schema_version == "2000m.v3.result.v1": freeze = data["protocolFreeze"] require(freeze["changedAfterLiveResults"] is False, "frozen protocol mutation is calibration-only and invalid in foundation fixtures") @@ -314,7 +324,7 @@ def validate_semantics(data: dict[str, Any], schema_version: str) -> None: require(data["anonymized"] is True, "visual package must be anonymized before blind review") require(data["mappingSealedBeforeReview"] is True, "blind label map must be sealed before review") for window in data["windows"]: - for key in ("seed", "captureCommand", "screenshotRef", "replayRef", "frameMetadataRef", "rubricMetadataRef", "fps", "frameCount", "inputSequenceRef", "stateChecksum", "frameChecksum"): + for key in ("seed", "captureCommand", "screenshotRef", "screenshotChecksum", "replayRef", "replayChecksum", "frameMetadataRef", "rubricMetadataRef", "fps", "frameCount", "inputSequenceRef", "stateChecksum", "frameChecksum"): require(key in window, f"visual package missing capture metadata {key}") require(is_plain_int(window["seed"]), "capture seed must be integer, not boolean") elif schema_version == "2000m.v3.manifest.v1": diff --git a/scripts/validate_v3_visual_packages.py b/scripts/validate_v3_visual_packages.py index 50130f8..37150da 100755 --- a/scripts/validate_v3_visual_packages.py +++ b/scripts/validate_v3_visual_packages.py @@ -41,13 +41,6 @@ def sha256_file(path: Path) -> str: return "sha256:" + h.hexdigest() -def sha256_pair(first: Path, second: Path) -> str: - h = hashlib.sha256() - h.update(first.read_bytes()) - h.update(second.read_bytes()) - return "sha256:" + h.hexdigest() - - def require(condition: bool, message: str) -> None: if not condition: raise SystemExit(message) @@ -104,7 +97,11 @@ def validate_package(path: Path) -> None: require(frames.get("fps") == item["fps"], f"frames fps mismatch for {frames_path}") require(frames.get("stateChecksum") == item["stateChecksum"], f"stateChecksum mismatch for {frames_path}") require(frames.get("frameChecksum") == item["frameChecksum"], f"frameChecksum mismatch for {frames_path}") - require(sha256_pair(screenshot, replay) == item["frameChecksum"], f"frameChecksum does not match screenshot+replay bytes for {frames_path}") + require(sha256_file(screenshot) == item["screenshotChecksum"], f"screenshotChecksum does not match screenshot bytes for {frames_path}") + require(sha256_file(replay) == item["replayChecksum"], f"replayChecksum does not match replay bytes for {frames_path}") + rubric = load_json(rubric_path) + if "frameChecksum" in rubric: + require(rubric["frameChecksum"] == item["frameChecksum"], f"rubric frameChecksum mismatch for {rubric_path}") require(sha256_file(rubric_path).startswith("sha256:"), "rubric metadata checksum computation failed") print(f"OK: v3 visual package {path.relative_to(ROOT) if path.is_relative_to(ROOT) else path}") diff --git a/v3/EVIDENCE_AND_CLAIMS.md b/v3/EVIDENCE_AND_CLAIMS.md index 57d18a3..712d689 100644 --- a/v3/EVIDENCE_AND_CLAIMS.md +++ b/v3/EVIDENCE_AND_CLAIMS.md @@ -20,6 +20,7 @@ For every scored generation in every lane, record: - prompt ref; - final model response ref; - stdout/stderr refs; +- runtime token telemetry when the runtime exposes reliable usage lines; - files changed or artifact refs; - build command and result; - test command and result; @@ -108,7 +109,7 @@ Allowed wording: > Repeatable workflow-value support candidate under this v3 scenario. -### Public benchmark support +### Externally reviewed support Reserve for larger campaigns with multiple seeds, frozen scenarios, public-safe evidence, and independent review. Do not use for private pilots. @@ -166,8 +167,25 @@ mechanical correctness product / visual artifact quality workflow resilience trajectory quality +token / cost efficiency evidence / recovery / governance quality claim boundary ``` Do not collapse these into a single feel-good narrative. + +When token/cost telemetry is present, the report should show: + +```text +tokens per generation +total tokens by seed/lane +total tokens by lane +tokens until first ranked mechanical ceiling +tokens per passed-AC improvement where meaningful +Lane B vs Lane A token delta +unavailable token/cost subfields +``` + +If the runtime exposes only total tokens, record only total tokens. Do not infer +input/output/cached/reasoning splits or dollar cost from logs that do not carry +those fields. diff --git a/v3/IMPLEMENTATION_PLAN.md b/v3/IMPLEMENTATION_PLAN.md index aa4f033..57ee6ec 100644 --- a/v3/IMPLEMENTATION_PLAN.md +++ b/v3/IMPLEMENTATION_PLAN.md @@ -130,6 +130,7 @@ Acceptance criteria: - Native visual packages are required for visual track ranking. - Missing visual packages block visual claims. - Decision thresholds are predeclared. +- Runtime token telemetry is parsed from source-labeled logs or receipts and reported separately from mechanical scoring. ## Gate 6 — private pilot @@ -145,6 +146,7 @@ context wipe: after first scored generation reviewer packet: one per pair, same phase visual windows: fixed before run claim ceiling: private directional signal only +token telemetry: source-labeled total tokens when reliable; split/cost fields nullable unless supplied by the runtime ``` ## Gate 7 — larger evidence campaign diff --git a/v3/MANIFEST_AND_CAPTURE.md b/v3/MANIFEST_AND_CAPTURE.md index b38c9fc..516c660 100644 --- a/v3/MANIFEST_AND_CAPTURE.md +++ b/v3/MANIFEST_AND_CAPTURE.md @@ -91,11 +91,18 @@ Required `frames.json` fields: "inputSequenceRef": "inputs.json", "stateChecksum": "sha256:...", "frameChecksum": "sha256:...", + "screenshotChecksum": "sha256:...", + "replayChecksum": "sha256:...", "events": ["..."], "warnings": [] } ``` +`frameChecksum` is the checksum declared by native frame metadata. It must not +be repurposed as a package-level checksum over screenshot/GIF bytes. Visual +packages record screenshot and replay byte checksums separately so the validator +can verify both the native capture metadata and the packaged media artifacts. + Capture commands must be allowed to fail honestly. If a capture fails, the result record must include the command, stdout/stderr refs, and failure reason. ## Standard capture windows diff --git a/v3/SCORING_MODEL.md b/v3/SCORING_MODEL.md index 3d9a5fb..6a4ab86 100644 --- a/v3/SCORING_MODEL.md +++ b/v3/SCORING_MODEL.md @@ -13,6 +13,33 @@ v3 reports four tracks separately: No single composite may hide these tracks. If a UI later renders a convenience summary, it must preserve every component and state that workflow score is not model intelligence. +Token/cost telemetry is a separate trajectory-efficiency input, not a +mechanical score component. A campaign may report total tokens by +seed/lane/generation, tokens until first ranked mechanical ceiling, and token +delta between lanes only when those fields are source-labeled and kept out of +mechanical pass counts. + +## Discrimination backlog + +Future frozen protocols should add neutral tracks that make ties less likely +without coupling the benchmark to any workflow product: + +- trajectory efficiency: generations until first valid ceiling, regressions, + and avoidable no-op generations; +- token/cost efficiency: source-labeled total tokens and cost only when a + versioned cost source exists; +- context-wipe/handoff: fresh-agent recovery from compact records without chat + history; +- plateau detection: stop/redesign recommendations when attempts stop moving; +- impossible/stale AC detection: scorer or requirement inspection instead of + blind retries; +- reviewer feedback resilience: exact-diagnostic parity plus correct handling + of reviewer corrections and regression traps; +- visual/product track: native capture validity as a prerequisite before any + visual ranking; +- hidden/regression traps: public-safe trap classes that are independent of + Open Scaffold, lane names, directory shape, or command vocabulary. + ## Draft track fields ### Mechanical @@ -56,6 +83,16 @@ No single composite may hide these tracks. If a UI later renders a convenience s - `evidence.requiredRefsMissing[]` - `evidence.claimBoundary` +### Runtime telemetry + +- `runtimeTelemetry.source.kind` +- `runtimeTelemetry.source.ref` +- `runtimeTelemetry.tokens.total` +- nullable `runtimeTelemetry.tokens.input/output/cachedInput/reasoning` +- `runtimeTelemetry.tokens.unavailableFields[]` +- nullable `runtimeTelemetry.cost.estimatedUsd` +- `runtimeTelemetry.cost.unavailableReason` + ## Blockers | Blocker | Mechanical | Visual | Workflow | Evidence/claim impact | @@ -66,6 +103,7 @@ No single composite may hide these tracks. If a UI later renders a convenience s | Frozen protocol changed after live results | affected run calibration-only | affected run calibration-only | affected run calibration-only | public support blocked | | Unsupported claim text | unaffected | unaffected | unaffected | evidence fails; public claims blocked | | Evidence volume without decision quality | unaffected | unaffected | no score credit | evidence may still pass only if compact and useful | +| Higher token/cost use | unaffected | unaffected unless the workflow track predeclares efficiency scoring | must be reported separately | cannot rescue tied or worse output | ## Claim ceiling diff --git a/v3/VISUAL_RUBRIC.md b/v3/VISUAL_RUBRIC.md index f9d5261..1ad885a 100644 --- a/v3/VISUAL_RUBRIC.md +++ b/v3/VISUAL_RUBRIC.md @@ -26,7 +26,7 @@ Every ranked visual package must validate against `v3/visual-package.schema.json - per-window frame metadata; - rubric record; - artifact digest captured before rendering; -- capture command and checksums for each window. +- capture command, native frame checksum, screenshot byte checksum, and replay byte checksum for each window. ## Capture windows @@ -62,6 +62,7 @@ The visual track is rank-blocked when: - capture command is missing or fails without valid rerun; - screenshot/replay/frame metadata refs are missing; - frame metadata omits seed, window, frame count, FPS, input ref, state checksum, or frame checksum; +- screenshot or replay byte checksums are missing or do not match packaged media; - fixed seeds/windows changed after live result inspection; - blind label map was opened before rating; - package/public record contains private local refs; diff --git a/v3/examples/valid/run-record.foundation.json b/v3/examples/valid/run-record.foundation.json index 4a48083..a3df86d 100644 --- a/v3/examples/valid/run-record.foundation.json +++ b/v3/examples/valid/run-record.foundation.json @@ -86,6 +86,31 @@ } ] }, + "runtimeTelemetry": { + "source": { + "kind": "codex-stderr", + "ref": "fixtures/codex-stderr.log", + "parser": "codex-cli-tokens-used-line-v1" + }, + "tokens": { + "total": 12345, + "input": null, + "output": null, + "cachedInput": null, + "reasoning": null, + "unavailableFields": [ + "input", + "output", + "cachedInput", + "reasoning" + ] + }, + "cost": { + "estimatedUsd": null, + "currency": null, + "unavailableReason": "Fixture has no versioned pricing table." + } + }, "finalRecommendation": { "decision": "redesign", "rationaleRefs": [ diff --git a/v3/examples/valid/visual-package.foundation.json b/v3/examples/valid/visual-package.foundation.json index 2285555..7c65c87 100644 --- a/v3/examples/valid/visual-package.foundation.json +++ b/v3/examples/valid/visual-package.foundation.json @@ -13,7 +13,9 @@ "seed": 1101, "captureCommand": "cargo run --quiet --bin capture -- --seed 1101 --window early-game --out visual-package/A/early-game", "screenshotRef": "visual-package/screenshots/A-early-game.png", + "screenshotChecksum": "sha256:2222222222222222222222222222222222222222222222222222222222222222", "replayRef": "visual-package/gifs/A-early-game.gif", + "replayChecksum": "sha256:3333333333333333333333333333333333333333333333333333333333333333", "frameMetadataRef": "visual-package/frames/A-early-game.frames.json", "rubricMetadataRef": "visual-package/frames/A-early-game.rubric-metadata.json", "fps": 30, diff --git a/v3/fixtures/visual/valid-artifact/capture.py b/v3/fixtures/visual/valid-artifact/capture.py index 024a9d5..8603a03 100755 --- a/v3/fixtures/visual/valid-artifact/capture.py +++ b/v3/fixtures/visual/valid-artifact/capture.py @@ -35,7 +35,12 @@ def main() -> int: screenshot = PNG_BYTES + f"\n# seed={args.seed} window={args.window}\n".encode() replay = GIF_BYTES + f"\n# seed={args.seed} window={args.window}\n".encode() state = {"seed": args.seed, "window": args.window, "fixture": "visual-capture", "tick": 120} - frame_checksum = sha256_bytes(screenshot + replay) + frame_checksum = sha256_json({ + "state": state, + "fps": 30, + "frameCount": 120, + "events": ["spawn", "turn"], + }) frames = { "schemaVersion": "2000m.capture.frames.v1", "seed": args.seed, diff --git a/v3/run-record.schema.json b/v3/run-record.schema.json index 4a252ce..4ba05e8 100644 --- a/v3/run-record.schema.json +++ b/v3/run-record.schema.json @@ -144,6 +144,9 @@ "evidence": { "$ref": "#/$defs/evidenceStatus" }, + "runtimeTelemetry": { + "$ref": "#/$defs/runtimeTelemetry" + }, "finalRecommendation": { "$ref": "#/$defs/finalRecommendation" } @@ -540,6 +543,133 @@ } } }, + "runtimeTelemetry": { + "type": "object", + "additionalProperties": false, + "required": [ + "source", + "tokens", + "cost" + ], + "properties": { + "source": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "ref", + "parser" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "codex-stderr", + "adapter-receipt", + "manual-entry", + "unknown" + ] + }, + "ref": { + "type": "string" + }, + "parser": { + "type": "string", + "minLength": 1 + } + } + }, + "tokens": { + "type": "object", + "additionalProperties": false, + "required": [ + "total", + "input", + "output", + "cachedInput", + "reasoning", + "unavailableFields" + ], + "properties": { + "total": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "input": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "output": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "cachedInput": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "reasoning": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "unavailableFields": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "total", + "input", + "output", + "cachedInput", + "reasoning" + ] + } + } + } + }, + "cost": { + "type": "object", + "additionalProperties": false, + "required": [ + "estimatedUsd", + "currency", + "unavailableReason" + ], + "properties": { + "estimatedUsd": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "currency": { + "type": [ + "string", + "null" + ] + }, + "unavailableReason": { + "type": "string" + } + } + } + } + }, "finalRecommendation": { "type": "object", "additionalProperties": false, diff --git a/v3/visual-package.schema.json b/v3/visual-package.schema.json index 7e0f3e4..6e6918e 100644 --- a/v3/visual-package.schema.json +++ b/v3/visual-package.schema.json @@ -123,7 +123,9 @@ "seed", "captureCommand", "screenshotRef", + "screenshotChecksum", "replayRef", + "replayChecksum", "frameMetadataRef", "rubricMetadataRef", "fps", @@ -155,10 +157,18 @@ "type": "string", "minLength": 1 }, + "screenshotChecksum": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, "replayRef": { "type": "string", "minLength": 1 }, + "replayChecksum": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, "frameMetadataRef": { "type": "string", "minLength": 1