Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 27 additions & 8 deletions tests/test_inventable_jetmod_instrumentation_status.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
Expand All @@ -21,23 +22,41 @@
}


def test_runner_writes_partial_and_refused_not_24jet_only():
result = subprocess.run(
[sys.executable, RUNNER], cwd=ROOT, capture_output=True, text=True, check=False
)
def _probe_snapshot():
return {
p.name: (p.read_bytes(), p.stat().st_mtime_ns)
for p in Path(PROBES).glob("*.json")
}


def test_runner_writes_partial_and_refused_not_24jet_only(tmp_path):
before = _probe_snapshot()
probes = tmp_path / "repo" / "docs" / "math_status_probes"
probes.mkdir(parents=True)
runner = probes / Path(RUNNER).name
shutil.copy2(RUNNER, runner)
# No old output is copied: success requires fresh receipts from the runner.
assert not list(probes.glob("*.json"))
try:
result = subprocess.run(
[sys.executable, str(runner)], cwd=probes.parent.parent,
capture_output=True, text=True, check=False, timeout=60,
)
finally:
assert _probe_snapshot() == before, "probe runner modified source receipts"
assert result.returncode == 0, result.stdout + result.stderr
assert "discharges=false" in result.stdout
for name, status in EXPECTED.items():
path = os.path.join(PROBES, name)
obj = json.load(open(path, encoding="utf-8"))
path = probes / name
obj = json.loads(path.read_text(encoding="utf-8"))
assert obj["status"] == status
assert obj["inventable_attempt_accepted"] is False
assert obj["discharges_OBL_H5_JETMOD"] is False
assert obj["lemma_closed"] is False
assert obj["certified_C_H"] is False
assert obj["prizes_solved"] == 0
index = json.load(
open(os.path.join(PROBES, "INVENTABLE_INSTRUMENTATION_STATUS_INDEX.json"), encoding="utf-8")
index = json.loads(
(probes / "INVENTABLE_INSTRUMENTATION_STATUS_INDEX.json").read_text(encoding="utf-8")
)
assert index["discharges_OBL_H5_JETMOD"] is False
assert index["lemma_closed"] is False
Expand Down
45 changes: 27 additions & 18 deletions tests/test_inventable_jetmod_probes.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@

import json
import os
from pathlib import Path
import shutil
import subprocess
import sys

Expand All @@ -28,20 +30,33 @@
}


def test_runner_writes_refused_receipts_only():
before = {
name: open(os.path.join(PROBES, name), "rb").read()
for name in EXPECTED
if os.path.isfile(os.path.join(PROBES, name))
def _probe_snapshot():
return {
p.name: (p.read_bytes(), p.stat().st_mtime_ns)
for p in Path(PROBES).glob("*.json")
}
result = subprocess.run(
[sys.executable, RUNNER], cwd=ROOT, capture_output=True, text=True, check=False
)


def test_runner_writes_refused_receipts_only(tmp_path):
before = _probe_snapshot()
probes = tmp_path / "repo" / "docs" / "math_status_probes"
probes.mkdir(parents=True)
runner = probes / Path(RUNNER).name
shutil.copy2(RUNNER, runner)
# No old output is copied: success requires fresh receipts from the runner.
assert not list(probes.glob("*.json"))
try:
result = subprocess.run(
[sys.executable, str(runner)], cwd=probes.parent.parent,
capture_output=True, text=True, check=False, timeout=60,
)
finally:
assert _probe_snapshot() == before, "probe runner modified source receipts"
assert result.returncode == 0, result.stdout + result.stderr
assert "discharges=false" in result.stdout
for name, status in EXPECTED.items():
path = os.path.join(PROBES, name)
obj = json.load(open(path, encoding="utf-8"))
path = probes / name
obj = json.loads(path.read_text(encoding="utf-8"))
assert obj["status"] == status
assert obj["inventable_attempt_accepted"] is False
assert obj["discharges_OBL_H5_JETMOD"] is False
Expand All @@ -58,8 +73,7 @@ def test_runner_writes_refused_receipts_only():
assert obj["roster_invented"] is False
assert "jet_roster" not in obj
assert "roster" not in obj
# re-run is allowed; flags must stay false
index = json.load(open(os.path.join(PROBES, "INVENTABLE_PROBES_INDEX.json"), encoding="utf-8"))
index = json.loads((probes / "INVENTABLE_PROBES_INDEX.json").read_text(encoding="utf-8"))
assert index["discharges_OBL_H5_JETMOD"] is False
assert index["discharges_lemma"] is False
assert index["lemma_closed"] is False
Expand All @@ -86,13 +100,8 @@ def test_math_status_check_validates_inventable_probes():


def test_negative_accepting_inventable_attempt_is_refused(tmp_path):
# mutate a receipt to accept inventable attempt → checker must fail
import shutil

# Mutate only a copied receipt; the published source is never rewritten.
dest = tmp_path / "repo"
# minimal: copy probes + docs/math_status + tools checker into temp layout
# Use in-place mutation of a copy of probes next to a fake root is hard;
# instead mutate real receipt temporarily is bad. Copy whole needed tree.
shutil.copytree(os.path.join(ROOT, "docs"), dest / "docs")
shutil.copytree(os.path.join(ROOT, "tools"), dest / "tools")
path = dest / "docs" / "math_status_probes" / "inventable_phi_bridge_ABSENT_receipt.json"
Expand Down
74 changes: 74 additions & 0 deletions tests/test_probe_runner_isolation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
"""Regression for probe-test isolation; no mathematical certification or status change."""
from __future__ import annotations

import importlib.util
from pathlib import Path
import shutil

import pytest

ROOT = Path(__file__).resolve().parents[1]
CASES = [
(
"test_inventable_jetmod_probes.py",
"inventable_jetmod_probes.py",
"test_runner_writes_refused_receipts_only",
8,
),
(
"test_inventable_jetmod_instrumentation_status.py",
"inventable_jetmod_instrumentation_status.py",
"test_runner_writes_partial_and_refused_not_24jet_only",
6,
),
]


def _load_case(tmp_path, monkeypatch, case, regress=False):
test_name, runner_name, function_name, output_count = case
source = tmp_path / "source" / "docs" / "math_status_probes"
source.mkdir(parents=True)
shutil.copy2(ROOT / "docs" / "math_status_probes" / runner_name, source / runner_name)
(source / "sentinel.json").write_text('{"published": true}\n', encoding="utf-8")
text = (ROOT / "tests" / test_name).read_text(encoding="utf-8")
if regress:
# Reintroduce the actual bug in a disposable copy, never in the checkout.
needle = "[sys.executable, str(runner)]"
assert text.count(needle) == 1
text = text.replace(needle, "[sys.executable, RUNNER]", 1)
path = tmp_path / test_name
path.write_text(text, encoding="utf-8")
spec = importlib.util.spec_from_file_location("_probe_isolation_case", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
monkeypatch.setattr(module, "PROBES", str(source))
monkeypatch.setattr(module, "RUNNER", str(source / runner_name))
return module, source, function_name, output_count


def _snapshot(directory):
return {
p.name: (p.read_bytes(), p.stat().st_mtime_ns)
for p in directory.iterdir() if p.is_file()
}


@pytest.mark.parametrize("case", CASES, ids=["probes", "instrumentation"])
def test_real_runner_test_preserves_source(tmp_path, monkeypatch, case):
module, source, function_name, output_count = _load_case(tmp_path, monkeypatch, case)
before = _snapshot(source)
work = tmp_path / "work"
getattr(module, function_name)(work)
assert _snapshot(source) == before
generated = work / "repo" / "docs" / "math_status_probes"
assert len(list(generated.glob("*.json"))) == output_count


@pytest.mark.parametrize("case", CASES, ids=["probes", "instrumentation"])
def test_source_path_regression_is_detected(tmp_path, monkeypatch, case):
module, source, function_name, _ = _load_case(tmp_path, monkeypatch, case, regress=True)
before = _snapshot(source)
with pytest.raises(AssertionError, match="probe runner modified source receipts"):
getattr(module, function_name)(tmp_path / "work")
assert _snapshot(source) != before # the control really exercised a source write
Loading