From 896ec8e2c3c944f26179ae20e2cfba094b759385 Mon Sep 17 00:00:00 2001 From: Glenn Neuber Date: Sat, 29 Aug 2026 15:42:31 +1000 Subject: [PATCH 1/3] preflight: generate a per-release 'what tested green' matrix Renders surface x capability from recorded preflight run artifacts: build identity, image size ladder, thinking on/off, output quality, the fp16 overflow canary, runner isolation. Generated, never hand-written. A hand-written green badge is worse than none -- it claims a surface was validated when nobody ran it, and nothing goes back to correct it later. Two properties that make it honest rather than decorative: - The six supported surfaces are listed explicitly, so one with no run appears as a full 'not run' row instead of vanishing. A missing row reads as 'not applicable', which is not the same claim and is the more dangerous of the two. mlx-cuda, rocm and cpu currently have no runs at all and now say so. - --version drops runs from other builds, so a release matrix cannot borrow a green cell from an older one. Filtered to 0.33.2-dynres every cell reads 'not run', which is exactly true: the image is still building. A group reports at its weakest check, so one skipped probe never reads as a pass. Attaching the matrix is added to the sync acceptance criteria, to be regenerated as each surface is preflighted. Co-Authored-By: Claude Fable 5 --- .../maxusai/tasks/upstream-sync-2026-08-28.md | 4 +- .../vision-suite/preflight/release_matrix.py | 104 ++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 docs/maxusai/vision-suite/preflight/release_matrix.py diff --git a/docs/maxusai/tasks/upstream-sync-2026-08-28.md b/docs/maxusai/tasks/upstream-sync-2026-08-28.md index 6002aced6e4..b16d6f1d5db 100644 --- a/docs/maxusai/tasks/upstream-sync-2026-08-28.md +++ b/docs/maxusai/tasks/upstream-sync-2026-08-28.md @@ -103,7 +103,9 @@ Glenn's call, not the merger's. builder, stamped `0.33.2-dynres-0-geaaf951`), then **full preflight re-measurement** (the pin moved), and update `expectations.toml` `payload_pin`. 7. ☑ Cut the GitHub Release for the fold tag and update the README's - **Current fold** pointer. Both are part of the fold, not follow-up: a + **Current fold** pointer, and attach the generated green matrix + (`preflight/release_matrix.py --version `) to the release notes — + regenerated after each surface is preflighted, never hand-edited. Both are part of the fold, not follow-up: a stale pointer is worse than none, because it claims a fixed point that no longer matches what `main` builds. 8. ☐ Metal half on the Apple host — deferred by Glenn, not a blocker here. diff --git a/docs/maxusai/vision-suite/preflight/release_matrix.py b/docs/maxusai/vision-suite/preflight/release_matrix.py new file mode 100644 index 00000000000..b033c73fcf0 --- /dev/null +++ b/docs/maxusai/vision-suite/preflight/release_matrix.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Render "what tested green" for a release, from preflight run artifacts. + + python3 release_matrix.py runs/*.json > matrix.md + +The point is that this is *generated*. A hand-written green badge is worse +than none: it claims a surface was validated when nobody ran it, and nothing +ever goes back to correct it. Every cell here comes from a recorded run, and +a surface with no run for the release reads "not run" rather than blank. + +Rows are surfaces (the preflight platform: cuda, mlx-cuda, mlx-metal, rocm, +cpu). Columns are what an operator actually wants to know held. +""" +import argparse +import glob +import json +import sys +from collections import defaultdict + +# Every surface the fork claims to support. Listed explicitly so one with no +# run still appears as a "not run" row instead of silently vanishing from the +# table -- a missing row reads as "not applicable", which is not the same +# thing and is the more dangerous of the two. +SURFACES = ["cuda", "mlx-cuda", "mlx-metal", "apple-silicon-mlx", "rocm", "cpu"] + +# check name -> the column an operator thinks in +GROUPS = [ + ("Build identity", {"version", "image_tag", "payload_pin", "go_patch_marker", + "payload_proof"}), + ("Image size ladder", {"token_ladder", "pinned_budget"}), + ("thinking on/off", {"think_format"}), + ("Output quality", {"text_baseline", "quality"}), + ("fp16 overflow canary", {"poison_probe"}), + ("Runner isolation", {"endpoint_exclusive"}), +] +RANK = {"FAIL": 0, "ERROR": 0, "CONTENTION": 1, "NEEDS_BASELINE": 1, + "SKIP": 2, "PASS": 3} +MARK = {"FAIL": "**FAIL**", "ERROR": "**ERROR**", "CONTENTION": "contended", + "NEEDS_BASELINE": "no baseline", "SKIP": "skipped", "PASS": "green"} + + +def worst(statuses): + """A group is only as good as its weakest check — never average them.""" + if not statuses: + return None + return min(statuses, key=lambda s: RANK.get(s, 1)) + + +def main(paths, version=None): + runs = [] + for p in paths: + try: + runs.append(json.load(open(p))) + except Exception as exc: # noqa: BLE001 + print(f"skipping {p}: {exc}", file=sys.stderr) + if not runs: + sys.exit("no readable run artifacts") + + if version: + # A release matrix must not borrow a green cell from a different + # build. Runs that do not match the release are dropped, and their + # surfaces then correctly report "not run". + runs = [r for r in runs + if r.get("meta", {}).get("version", "").startswith(version)] + + by_surface = defaultdict(list) + for r in runs: + by_surface[r.get("meta", {}).get("platform") or "unknown"].append(r) + surfaces = SURFACES + [s for s in sorted(by_surface) if s not in SURFACES] + + print("| surface | " + " | ".join(g for g, _ in GROUPS) + " | measured on |") + print("|---" * (len(GROUPS) + 2) + "|") + for surface in surfaces: + if not by_surface.get(surface): + print(f"| **{surface}** | " + " | ".join(["not run"] * len(GROUPS)) + + " | — |") + continue + # newest run wins for a surface + run = max(by_surface[surface], + key=lambda r: r.get("meta", {}).get("started_utc", "")) + results = run.get("results", []) + cells = [] + for _, names in GROUPS: + got = [r["status"] for r in results if r.get("check") in names] + w = worst(got) + cells.append(MARK.get(w, "not run") if w else "not run") + meta = run.get("meta", {}) + cells.append(f"`{meta.get('version', '?')}`") + print(f"| **{surface}** | " + " | ".join(cells) + " |") + + print() + print("Generated by `release_matrix.py` from recorded preflight runs. " + "A surface with no run for this release reads *not run* — absence is " + "shown, never assumed green. A group is reported at its weakest " + "check, so one skipped probe does not read as a pass.") + + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("runs", nargs="*", help="preflight run JSONs") + ap.add_argument("--version", help="only count runs whose version starts " + "with this, e.g. 0.33.2-dynres") + a = ap.parse_args() + main(a.runs or sorted(glob.glob("runs/*.json")), a.version) From eff142f6f95ef2d8b402e3c7222daf293045b174 Mon Sep 17 00:00:00 2001 From: Glenn Neuber Date: Sat, 29 Aug 2026 21:32:49 +1000 Subject: [PATCH 2/3] preflight: split the ladder column, and read both check names Two fixes to the generated matrix. The 'Image size ladder' column bundled token_ladder with the pinned-budget probe and reported the group at its weakest check, so CUDA read 'skipped' even though the ladder passed on all three arches -- the skips belong to the narrower pinned probe, which only nemotron has an expectation for. They answer different questions and now get their own columns. It also maps both the new pinned_image_token_budget name and the old pinned_budget, because runs recorded before the rename carry the old one and a matrix that silently stopped seeing the check would render it as green rather than missing. Co-Authored-By: Claude Fable 5 --- docs/maxusai/vision-suite/preflight/release_matrix.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/maxusai/vision-suite/preflight/release_matrix.py b/docs/maxusai/vision-suite/preflight/release_matrix.py index b033c73fcf0..543830c4ef5 100644 --- a/docs/maxusai/vision-suite/preflight/release_matrix.py +++ b/docs/maxusai/vision-suite/preflight/release_matrix.py @@ -27,7 +27,10 @@ GROUPS = [ ("Build identity", {"version", "image_tag", "payload_pin", "go_patch_marker", "payload_proof"}), - ("Image size ladder", {"token_ladder", "pinned_budget"}), + ("Image size ladder", {"token_ladder"}), + # Both names: runs recorded before the rename carry "pinned_budget", and a + # matrix that silently stopped seeing this check would read as green. + ("Pinned image budget", {"pinned_image_token_budget", "pinned_budget"}), ("thinking on/off", {"think_format"}), ("Output quality", {"text_baseline", "quality"}), ("fp16 overflow canary", {"poison_probe"}), From 41b3394cc702b8bb5ffc01b9393562b9d059b4ab Mon Sep 17 00:00:00 2001 From: Glenn Neuber Date: Sat, 29 Aug 2026 21:55:39 +1000 Subject: [PATCH 3/3] preflight: treat "not applicable" as neutral in the matrix, not as weak The harness deliberately separates two skips that mean different things: an arch structurally unable to run a probe, and one nobody has measured. Both arrive as SKIP and the distinction survives only in the summary text, so the matrix was collapsing them and reporting a column as "skipped" when two of three arches had passed and the third simply cannot run it. N/A is now recognised from the summary and treated as NEUTRAL -- dropped from the worst-wins comparison, reported only when every cell is N/A. Ranking it weak reports "n/a" for a column that passed; ranking it strong hides a real gap behind it; neither is honest. Checked in both directions on real runs: the post-fix CUDA run reads green, and the run from before the gaps were closed still reads "skipped". The change distinguishes not-applicable from unmeasured rather than making things look green. Co-Authored-By: Claude Fable 5 --- .../vision-suite/preflight/release_matrix.py | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/docs/maxusai/vision-suite/preflight/release_matrix.py b/docs/maxusai/vision-suite/preflight/release_matrix.py index 543830c4ef5..a2910c63988 100644 --- a/docs/maxusai/vision-suite/preflight/release_matrix.py +++ b/docs/maxusai/vision-suite/preflight/release_matrix.py @@ -37,16 +37,42 @@ ("Runner isolation", {"endpoint_exclusive"}), ] RANK = {"FAIL": 0, "ERROR": 0, "CONTENTION": 1, "NEEDS_BASELINE": 1, - "SKIP": 2, "PASS": 3} + "SKIP": 2, "N/A": 3, "PASS": 4} MARK = {"FAIL": "**FAIL**", "ERROR": "**ERROR**", "CONTENTION": "contended", - "NEEDS_BASELINE": "no baseline", "SKIP": "skipped", "PASS": "green"} + "NEEDS_BASELINE": "no baseline", "SKIP": "skipped", "N/A": "n/a", + "PASS": "green"} + + +def effective(r): + """A skip that says "does not apply" is NOT a coverage gap. + + The harness deliberately separates the two (checks.py: saying "not yet + measured" about something structurally unable to move sends someone off to + measure it). Both arrive here as SKIP, so the distinction only survives in + the summary text. Collapsing them would make an architecture that CANNOT + have the probe drag a column down as if nobody had run it -- understating + coverage, which is its own kind of dishonesty. + """ + if r.get("status") == "SKIP" and "does not apply" in (r.get("summary") or ""): + return "N/A" + return r.get("status") def worst(statuses): - """A group is only as good as its weakest check — never average them.""" + """A group is only as good as its weakest check — never average them. + + N/A is NEUTRAL, not weak: an architecture structurally unable to run a + probe should not drag down the ones that ran it and passed. It is dropped + from the comparison, and only reported when every cell is N/A. Ranking it + as weak would report "n/a" for a column two arches actually passed; + ranking it as strong would hide a real gap behind it. + """ if not statuses: return None - return min(statuses, key=lambda s: RANK.get(s, 1)) + scored = [s for s in statuses if s != "N/A"] + if not scored: + return "N/A" + return min(scored, key=lambda s: RANK.get(s, 1)) def main(paths, version=None): @@ -84,7 +110,7 @@ def main(paths, version=None): results = run.get("results", []) cells = [] for _, names in GROUPS: - got = [r["status"] for r in results if r.get("check") in names] + got = [effective(r) for r in results if r.get("check") in names] w = worst(got) cells.append(MARK.get(w, "not run") if w else "not run") meta = run.get("meta", {})