REVIEW ONLY — grading, key-resolution and diagnostic fixes from #50, sized for a machine reviewer

campaign3_blind/grade_blind.py

@@ -48,6 +48,24 @@
 
 HERE = Path(__file__).resolve().parent
 KEYS = HERE / "keys"
+PROBLEMS = HERE / "problems"
+
+
+def probe_exclusions(problem_id: str, side: str):
+    """Regions removed from a subdomain's probe grid, from the PUBLIC spec.
+
+    Deliberately not read from the key: the exclusion is already printed in the
+    task text handed to the agent, so it is public information and reading it
+    here keeps the sealed keys sealed.
+    """
+    spec = PROBLEMS / problem_id / "spec_public.json"
+    if not spec.is_file():
+        return None
+    try:
+        return json.loads(spec.read_text()).get(
+            f"probe_{side.lower()}_exclude")
+    except (json.JSONDecodeError, OSError):
+        return None
 
 x, y, z = sp.symbols("x y z", real=True)
 _SYMS = {"x": x, "y": y, "z": z}
@@ -87,16 +105,40 @@ def assert_probe_grid_incommensurate(dim: int, mesh_N) -> None:
 
 
 # ── the grader's own evaluation set ───────────────────────────────────────
-def probe_grid(dim: int, bounds=None):
+def probe_grid(dim: int, bounds=None, exclude=None):
     """Cell-centred grid, last index varying fastest. Never coincides with a
-    mesh node at any prescribed level."""
+    mesh node at any prescribed level.
+
+    `exclude` removes every point lying strictly inside any of the given
+    axis-aligned boxes, each written as [(lo, hi), ...] per axis. A subdomain
+    is not always a rectangle: D5's subdomain A is the unit square minus the
+    other subdomain minus a notch, and its task text says so and states the
+    1331 points that remain. Without this the grader built the full 1936-point
+    rectangle and rejected a correct submission with "expected 1936 probe
+    points, got 1331" — the task was right and the grader was wrong.
+
+    The exclusion is public by construction: it is printed in the task text the
+    agent is given, so reading it costs no secrecy. It is taken from the public
+    spec rather than the key precisely so that no key has to be reopened.
+    """
     M = PROBE_M[dim]
     b = bounds or [(0.0, 1.0)] * dim
     axes = [[lo + (i + 0.5) * (hi - lo) / M for i in range(M)] for lo, hi in b]
     pts = [()]
     for ax in axes:
         pts = [p + (v,) for p in pts for v in ax]
-    return pts
+    if not exclude:
+        return pts
+
+    def inside(p, box):
+        return all(lo < c < hi for c, (lo, hi) in zip(p, box))
+
+    boxes = [[tuple(axis) for axis in box] for box in exclude]
+    for box in boxes:
+        if len(box) != dim:
+            raise ValueError(
+                f"exclusion box has {len(box)} axes but the problem is {dim}D")
+    return [p for p in pts if not any(inside(p, box) for box in boxes)]
 
 
 def subdomain_bounds(key: dict, side: str, dim: int):
@@ -306,7 +348,9 @@ def grade_run(run_dir: Path, problem_id: str) -> dict:
                         "observed_order": None, "note": f"{path.name}: {why}"}
             bounds = (subdomain_bounds(key, side, dim) if coupled
                       else [(0.0, 1.0)] * dim)
-            good, why = matches_probe_grid(pts, probe_grid(dim, bounds))
+            exclude = probe_exclusions(problem_id, side) if coupled else None
+            good, why = matches_probe_grid(
+                pts, probe_grid(dim, bounds, exclude))
             if not good:
                 return {**out, "outcome": "INVALID_SUBMISSION",
                         "observed_order": None, "note": f"{path.name}: {why}"}

campaign3_blind/problems/B1/task.txt

@@ -16,7 +16,7 @@ probe point and write one CSV file `solution_level<k>.csv` (k = 1,2,3,...)
 with a header line and one row per probe point, in the order defined below:
     x, y, u
 
-PROBE POINTS: the 1024 points given by x = (i_x+0.5)/32; y = (i_y+0.5)/32, for i_x, i_y = 0, 1, ..., 31 independently, ordered with the last index varying fastest
+PROBE POINTS: the 1936 points given by x = (i_x+0.5)/44; y = (i_y+0.5)/44, for i_x, i_y = 0, 1, ..., 43 independently, ordered with the last index varying fastest
 Evaluate (interpolate) your finite element solution at these points. They are
 deliberately not mesh nodes. Write full precision; do not round; every probe
 point must appear exactly once, and no other points may appear.

campaign3_blind/problems/B2/task.txt

@@ -16,7 +16,7 @@ probe point and write one CSV file `solution_level<k>.csv` (k = 1,2,3,...)
 with a header line and one row per probe point, in the order defined below:
     x, y, u
 
-PROBE POINTS: the 1024 points given by x = (i_x+0.5)/32; y = (i_y+0.5)/32, for i_x, i_y = 0, 1, ..., 31 independently, ordered with the last index varying fastest
+PROBE POINTS: the 1936 points given by x = (i_x+0.5)/44; y = (i_y+0.5)/44, for i_x, i_y = 0, 1, ..., 43 independently, ordered with the last index varying fastest
 Evaluate (interpolate) your finite element solution at these points. They are
 deliberately not mesh nodes. Write full precision; do not round; every probe
 point must appear exactly once, and no other points may appear.

campaign3_blind/problems/B3/task.txt

@@ -16,7 +16,7 @@ probe point and write one CSV file `solution_level<k>.csv` (k = 1,2,3,...)
 with a header line and one row per probe point, in the order defined below:
     x, y, u
 
-PROBE POINTS: the 1024 points given by x = (i_x+0.5)/32; y = (i_y+0.5)/32, for i_x, i_y = 0, 1, ..., 31 independently, ordered with the last index varying fastest
+PROBE POINTS: the 1936 points given by x = (i_x+0.5)/44; y = (i_y+0.5)/44, for i_x, i_y = 0, 1, ..., 43 independently, ordered with the last index varying fastest
 Evaluate (interpolate) your finite element solution at these points. They are
 deliberately not mesh nodes. Write full precision; do not round; every probe
 point must appear exactly once, and no other points may appear.

campaign3_blind/problems/B4/task.txt

@@ -16,7 +16,7 @@ probe point and write one CSV file `solution_level<k>.csv` (k = 1,2,3,...)
 with a header line and one row per probe point, in the order defined below:
     x, y, z, u
 
-PROBE POINTS: the 4096 points given by x = (i_x+0.5)/16; y = (i_y+0.5)/16; z = (i_z+0.5)/16, for i_x, i_y, i_z = 0, 1, ..., 15 independently, ordered with the last index varying fastest
+PROBE POINTS: the 9261 points given by x = (i_x+0.5)/21; y = (i_y+0.5)/21; z = (i_z+0.5)/21, for i_x, i_y, i_z = 0, 1, ..., 20 independently, ordered with the last index varying fastest
 Evaluate (interpolate) your finite element solution at these points. They are
 deliberately not mesh nodes. Write full precision; do not round; every probe
 point must appear exactly once, and no other points may appear.

campaign3_blind/problems/B5/task.txt

@@ -16,7 +16,7 @@ probe point and write one CSV file `solution_level<k>.csv` (k = 1,2,3,...)
 with a header line and one row per probe point, in the order defined below:
     x, y, ux, uy
 
-PROBE POINTS: the 1024 points given by x = (i_x+0.5)/32; y = (i_y+0.5)/32, for i_x, i_y = 0, 1, ..., 31 independently, ordered with the last index varying fastest
+PROBE POINTS: the 1936 points given by x = (i_x+0.5)/44; y = (i_y+0.5)/44, for i_x, i_y = 0, 1, ..., 43 independently, ordered with the last index varying fastest
 Evaluate (interpolate) your finite element solution at these points. They are
 deliberately not mesh nodes. Write full precision; do not round; every probe
 point must appear exactly once, and no other points may appear.

campaign3_blind/problems/B6/task.txt

@@ -16,7 +16,7 @@ probe point and write one CSV file `solution_level<k>.csv` (k = 1,2,3,...)
 with a header line and one row per probe point, in the order defined below:
     x, y, z, ux, uy, uz
 
-PROBE POINTS: the 4096 points given by x = (i_x+0.5)/16; y = (i_y+0.5)/16; z = (i_z+0.5)/16, for i_x, i_y, i_z = 0, 1, ..., 15 independently, ordered with the last index varying fastest
+PROBE POINTS: the 9261 points given by x = (i_x+0.5)/21; y = (i_y+0.5)/21; z = (i_z+0.5)/21, for i_x, i_y, i_z = 0, 1, ..., 20 independently, ordered with the last index varying fastest
 Evaluate (interpolate) your finite element solution at these points. They are
 deliberately not mesh nodes. Write full precision; do not round; every probe
 point must appear exactly once, and no other points may appear.

campaign3_blind/problems/B7/task.txt

@@ -16,7 +16,7 @@ probe point and write one CSV file `solution_level<k>.csv` (k = 1,2,3,...)
 with a header line and one row per probe point, in the order defined below:
     x, y, ux, uy
 
-PROBE POINTS: the 1024 points given by x = (i_x+0.5)/32; y = (i_y+0.5)/32, for i_x, i_y = 0, 1, ..., 31 independently, ordered with the last index varying fastest
+PROBE POINTS: the 1936 points given by x = (i_x+0.5)/44; y = (i_y+0.5)/44, for i_x, i_y = 0, 1, ..., 43 independently, ordered with the last index varying fastest
 Evaluate (interpolate) your finite element solution at these points. They are
 deliberately not mesh nodes. Write full precision; do not round; every probe
 point must appear exactly once, and no other points may appear.

scripts/audit_quoted_diagnostics.py

@@ -72,7 +72,16 @@
 # audit reports UNKNOWN for that backend rather than guessing.
 SOURCE_HINTS: dict[str, list[str]] = {
     # 4C: the source tree is READ ONLY — we only ever grep it.
-    "fourc": ["/home/alexander/4C/src", "/home/alexander/4C/tests"],
+    #
+    # `apps/` was missing and it is where the executable's own banners live.
+    # 4C_global_full_main.cpp holds printf("processor %d finished normally\n"),
+    # so the exit line every 4C fixture greps for was invisible to this audit
+    # and two entries quoting it were reported as fabricated diagnostics. 16
+    # files; the entry point of the shipped binary is not an optional part of
+    # 4C's source. `unittests/` is deliberately NOT added: a string that exists
+    # only in a test's expected output is not evidence the solver emits it.
+    "fourc": ["/home/alexander/4C/src", "/home/alexander/4C/apps",
+              "/home/alexander/4C/tests"],
     # FEBio is installed as a BINARY with no source tree — `/opt/febio` and
     # `/usr/local/febio` are both absent on this host, so the audit reported
     # UNKNOWN for every FEBio claim. The real install is below, and a binary is
@@ -602,6 +611,34 @@ def cpython_runtime(package_dirs: list[Path]) -> list[Path]:
                     real = cand.resolve()
                     if real.is_file() and real not in out:
                         out.append(real)
+                # A VENV DOES NOT CONTAIN ITS OWN libpython. It contains a
+                # pyvenv.cfg naming the base interpreter, and on this host that
+                # is a uv-managed CPython under ~/.local/share/python — so the
+                # glob above found nothing and libpython was absent from the
+                # corpus entirely.
+                #
+                # Measured: `float() argument must be a string or a real
+                # number, not 'complex'`, reproduced live in one line
+                # (lil_matrix()[0,0] = 1j), scored 0 hits and was reported as a
+                # fabricated skfem diagnostic. It is in
+                # cpython-3.12.13/lib/libpython3.12.so, 1 hit. Same class of
+                # instrument fault as the FEBio symlink and the missing -a:
+                # a corpus that cannot answer is not evidence of absence.
+                for cfg in (libdir / "pyvenv.cfg",
+                            libdir.parent / "pyvenv.cfg"):
+                    try:
+                        text = cfg.read_text(errors="ignore")
+                    except OSError:
+                        continue
+                    for line in text.splitlines():
+                        k, _, v = line.partition("=")
+                        if k.strip() != "home":
+                            continue
+                        base = Path(v.strip()).parent
+                        for cand in sorted(base.glob("lib/libpython3*.so*")):
+                            real = cand.resolve()
+                            if real.is_file() and real not in out:
+                                out.append(real)
                 break
     return out
 

scripts/run_tier2_fixtures.py

@@ -278,6 +278,25 @@ def _eval_fixture(fixture_dir: Path,
 
     # Build env (per-backend defaults).
     env = os.environ.copy()
+    # WHERE THE CHECKOUT IS, for fixtures that audit the catalog rather than a
+    # solver. In place they find it by walking up from __file__, but the
+    # mutation harness stages a copy into a scratch tree that has no such
+    # ancestor, so the walk fails and the fixture aborts with
+    # FIXTURE_ABORT=no_oasis_checkout — which the harness scores
+    # VACUOUS_BASELINE, i.e. "this verdict would mean nothing".
+    #
+    # Measured before this line: 7 of the 11 Kratos fixtures that ship a
+    # `_mutation` block reported VACUOUS_BASELINE on every run, so the ledger
+    # carried NO machine discrimination evidence for them even though each had
+    # been proved KILLED by hand with OASIS_REPO exported on the command line.
+    # The runner knows where the checkout is; the staged fixture cannot. It is
+    # the runner's job to say so.
+    #
+    # Deliberately does NOT override an OASIS_REPO the caller already set, and
+    # deliberately points at the REAL checkout rather than the scratch copy:
+    # these fixtures audit the SHIPPED catalog, and their mutation lives in
+    # their own source, not in the catalog.
+    env.setdefault("OASIS_REPO", str(REPO_ROOT))
     extra_env = meta.get("env", {})
     if isinstance(extra_env, dict):
         for k, v in extra_env.items():