From 8d023a9a59a1c9c99c8120f5ec1e5505a84d3b9f Mon Sep 17 00:00:00 2001 From: Dylan Roy Date: Wed, 23 Sep 2026 15:00:50 -0500 Subject: [PATCH] Isolate JETMOD probe tests from tracked receipts Copy only each runner into per-test temporary output trees. Check source JSON bytes and modification times in finally blocks. Add four real-runner regression cases, including reintroduced source-path mutations in disposable copies. No published receipts, scientific flags, source exports, or production runners changed. Author: OpenAI / ChatGPT. Initial local validation is syntax-only; full execution is delegated to existing repository CI and is not preclaimed. --- ...nventable_jetmod_instrumentation_status.py | 35 +++++++-- tests/test_inventable_jetmod_probes.py | 45 ++++++----- tests/test_probe_runner_isolation.py | 74 +++++++++++++++++++ 3 files changed, 128 insertions(+), 26 deletions(-) create mode 100644 tests/test_probe_runner_isolation.py diff --git a/tests/test_inventable_jetmod_instrumentation_status.py b/tests/test_inventable_jetmod_instrumentation_status.py index d63f08330..3902be7b6 100644 --- a/tests/test_inventable_jetmod_instrumentation_status.py +++ b/tests/test_inventable_jetmod_instrumentation_status.py @@ -3,6 +3,7 @@ import json import os +from pathlib import Path import shutil import subprocess import sys @@ -21,23 +22,41 @@ } -def test_runner_writes_partial_and_refused_not_24jet_only(): - result = subprocess.run( - [sys.executable, RUNNER], cwd=ROOT, capture_output=True, text=True, check=False - ) +def _probe_snapshot(): + return { + p.name: (p.read_bytes(), p.stat().st_mtime_ns) + for p in Path(PROBES).glob("*.json") + } + + +def test_runner_writes_partial_and_refused_not_24jet_only(tmp_path): + before = _probe_snapshot() + probes = tmp_path / "repo" / "docs" / "math_status_probes" + probes.mkdir(parents=True) + runner = probes / Path(RUNNER).name + shutil.copy2(RUNNER, runner) + # No old output is copied: success requires fresh receipts from the runner. + assert not list(probes.glob("*.json")) + try: + result = subprocess.run( + [sys.executable, str(runner)], cwd=probes.parent.parent, + capture_output=True, text=True, check=False, timeout=60, + ) + finally: + assert _probe_snapshot() == before, "probe runner modified source receipts" assert result.returncode == 0, result.stdout + result.stderr assert "discharges=false" in result.stdout for name, status in EXPECTED.items(): - path = os.path.join(PROBES, name) - obj = json.load(open(path, encoding="utf-8")) + path = probes / name + obj = json.loads(path.read_text(encoding="utf-8")) assert obj["status"] == status assert obj["inventable_attempt_accepted"] is False assert obj["discharges_OBL_H5_JETMOD"] is False assert obj["lemma_closed"] is False assert obj["certified_C_H"] is False assert obj["prizes_solved"] == 0 - index = json.load( - open(os.path.join(PROBES, "INVENTABLE_INSTRUMENTATION_STATUS_INDEX.json"), encoding="utf-8") + index = json.loads( + (probes / "INVENTABLE_INSTRUMENTATION_STATUS_INDEX.json").read_text(encoding="utf-8") ) assert index["discharges_OBL_H5_JETMOD"] is False assert index["lemma_closed"] is False diff --git a/tests/test_inventable_jetmod_probes.py b/tests/test_inventable_jetmod_probes.py index e8003f9d4..61ff14cd3 100644 --- a/tests/test_inventable_jetmod_probes.py +++ b/tests/test_inventable_jetmod_probes.py @@ -3,6 +3,8 @@ import json import os +from pathlib import Path +import shutil import subprocess import sys @@ -28,20 +30,33 @@ } -def test_runner_writes_refused_receipts_only(): - before = { - name: open(os.path.join(PROBES, name), "rb").read() - for name in EXPECTED - if os.path.isfile(os.path.join(PROBES, name)) +def _probe_snapshot(): + return { + p.name: (p.read_bytes(), p.stat().st_mtime_ns) + for p in Path(PROBES).glob("*.json") } - result = subprocess.run( - [sys.executable, RUNNER], cwd=ROOT, capture_output=True, text=True, check=False - ) + + +def test_runner_writes_refused_receipts_only(tmp_path): + before = _probe_snapshot() + probes = tmp_path / "repo" / "docs" / "math_status_probes" + probes.mkdir(parents=True) + runner = probes / Path(RUNNER).name + shutil.copy2(RUNNER, runner) + # No old output is copied: success requires fresh receipts from the runner. + assert not list(probes.glob("*.json")) + try: + result = subprocess.run( + [sys.executable, str(runner)], cwd=probes.parent.parent, + capture_output=True, text=True, check=False, timeout=60, + ) + finally: + assert _probe_snapshot() == before, "probe runner modified source receipts" assert result.returncode == 0, result.stdout + result.stderr assert "discharges=false" in result.stdout for name, status in EXPECTED.items(): - path = os.path.join(PROBES, name) - obj = json.load(open(path, encoding="utf-8")) + path = probes / name + obj = json.loads(path.read_text(encoding="utf-8")) assert obj["status"] == status assert obj["inventable_attempt_accepted"] is False assert obj["discharges_OBL_H5_JETMOD"] is False @@ -58,8 +73,7 @@ def test_runner_writes_refused_receipts_only(): assert obj["roster_invented"] is False assert "jet_roster" not in obj assert "roster" not in obj - # re-run is allowed; flags must stay false - index = json.load(open(os.path.join(PROBES, "INVENTABLE_PROBES_INDEX.json"), encoding="utf-8")) + index = json.loads((probes / "INVENTABLE_PROBES_INDEX.json").read_text(encoding="utf-8")) assert index["discharges_OBL_H5_JETMOD"] is False assert index["discharges_lemma"] is False assert index["lemma_closed"] is False @@ -86,13 +100,8 @@ def test_math_status_check_validates_inventable_probes(): def test_negative_accepting_inventable_attempt_is_refused(tmp_path): - # mutate a receipt to accept inventable attempt → checker must fail - import shutil - + # Mutate only a copied receipt; the published source is never rewritten. dest = tmp_path / "repo" - # minimal: copy probes + docs/math_status + tools checker into temp layout - # Use in-place mutation of a copy of probes next to a fake root is hard; - # instead mutate real receipt temporarily is bad. Copy whole needed tree. shutil.copytree(os.path.join(ROOT, "docs"), dest / "docs") shutil.copytree(os.path.join(ROOT, "tools"), dest / "tools") path = dest / "docs" / "math_status_probes" / "inventable_phi_bridge_ABSENT_receipt.json" diff --git a/tests/test_probe_runner_isolation.py b/tests/test_probe_runner_isolation.py new file mode 100644 index 000000000..ab21739bc --- /dev/null +++ b/tests/test_probe_runner_isolation.py @@ -0,0 +1,74 @@ +"""Regression for probe-test isolation; no mathematical certification or status change.""" +from __future__ import annotations + +import importlib.util +from pathlib import Path +import shutil + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +CASES = [ + ( + "test_inventable_jetmod_probes.py", + "inventable_jetmod_probes.py", + "test_runner_writes_refused_receipts_only", + 8, + ), + ( + "test_inventable_jetmod_instrumentation_status.py", + "inventable_jetmod_instrumentation_status.py", + "test_runner_writes_partial_and_refused_not_24jet_only", + 6, + ), +] + + +def _load_case(tmp_path, monkeypatch, case, regress=False): + test_name, runner_name, function_name, output_count = case + source = tmp_path / "source" / "docs" / "math_status_probes" + source.mkdir(parents=True) + shutil.copy2(ROOT / "docs" / "math_status_probes" / runner_name, source / runner_name) + (source / "sentinel.json").write_text('{"published": true}\n', encoding="utf-8") + text = (ROOT / "tests" / test_name).read_text(encoding="utf-8") + if regress: + # Reintroduce the actual bug in a disposable copy, never in the checkout. + needle = "[sys.executable, str(runner)]" + assert text.count(needle) == 1 + text = text.replace(needle, "[sys.executable, RUNNER]", 1) + path = tmp_path / test_name + path.write_text(text, encoding="utf-8") + spec = importlib.util.spec_from_file_location("_probe_isolation_case", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "PROBES", str(source)) + monkeypatch.setattr(module, "RUNNER", str(source / runner_name)) + return module, source, function_name, output_count + + +def _snapshot(directory): + return { + p.name: (p.read_bytes(), p.stat().st_mtime_ns) + for p in directory.iterdir() if p.is_file() + } + + +@pytest.mark.parametrize("case", CASES, ids=["probes", "instrumentation"]) +def test_real_runner_test_preserves_source(tmp_path, monkeypatch, case): + module, source, function_name, output_count = _load_case(tmp_path, monkeypatch, case) + before = _snapshot(source) + work = tmp_path / "work" + getattr(module, function_name)(work) + assert _snapshot(source) == before + generated = work / "repo" / "docs" / "math_status_probes" + assert len(list(generated.glob("*.json"))) == output_count + + +@pytest.mark.parametrize("case", CASES, ids=["probes", "instrumentation"]) +def test_source_path_regression_is_detected(tmp_path, monkeypatch, case): + module, source, function_name, _ = _load_case(tmp_path, monkeypatch, case, regress=True) + before = _snapshot(source) + with pytest.raises(AssertionError, match="probe runner modified source receipts"): + getattr(module, function_name)(tmp_path / "work") + assert _snapshot(source) != before # the control really exercised a source write