diff --git a/.github/workflows/c8-cap-failure-coefficient-dims.yml b/.github/workflows/c8-cap-failure-coefficient-dims.yml index afb65fea4..fe18ccae5 100644 --- a/.github/workflows/c8-cap-failure-coefficient-dims.yml +++ b/.github/workflows/c8-cap-failure-coefficient-dims.yml @@ -5,6 +5,10 @@ name: C8 cap-failure coefficient in every dimension (certified replay) paths: - 'frontiers/c8_cap_failure_coefficient_dims_20261001/**' - '.github/workflows/c8-cap-failure-coefficient-dims.yml' + - 'tools/**' + - 'tools.py' + - 'tests/test_check_mode_workflows.py' + - 'tests/test_legacy_workflow_triggers.py' - 'imports/lifetime_parent_20260925/UNIFORM_MATRIX_CAP_AND_LIFETIME.md' - 'imports/lifetime_parent_20260925/MARKED_CYLINDER_CAP_PROOF.md' - 'frontiers/spectral_cluster_closure_20260929/PROOF.md' @@ -24,6 +28,13 @@ jobs: - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 with: python-version: '3.11.16' + - name: Check trigger coverage and the complete shell rejection contract in both modes + env: + CHECK_MODE_TEST_FAMILY: 'c8-cap-failure-coefficient-dims' + PYTHONDONTWRITEBYTECODE: '1' + run: | + python -B -S tests/test_check_mode_workflows.py + python -B -O -S tests/test_check_mode_workflows.py - name: Verify manifest and main-resident pins, replay the certified values in both interpreter modes, reject mutants env: PYTHONDONTWRITEBYTECODE: '1' @@ -51,17 +62,8 @@ jobs: blob = hashlib.sha1(b'blob %d\0' % len(data) + data).hexdigest() if len(data) != entry['bytes'] or hashlib.sha256(data).hexdigest() != entry['sha256'] or blob != entry['git_blob']: raise ValueError('consumed source identity mismatch on this tree: ' + entry['path']) - script = str(root / 'cap_coefficient_dims.py') - for flags in (['-B', '-S'], ['-B', '-O', '-S']): - out = subprocess.run([sys.executable, *flags, script, '--check'], cwd=root, capture_output=True, timeout=600) - if out.returncode != 0: - raise RuntimeError('check failed under ' + ' '.join(flags) + ': ' + out.stdout.decode()[-400:] + out.stderr.decode()[-400:]) - print(flags, out.stdout.decode().strip()[-300:]) - for flags in (['-B', '-S'], ['-B', '-O', '-S']): - for m in ('no-y-tail', 'r3-one', 'no-bp', 'frob-omega-off'): - rc = subprocess.run([sys.executable, *flags, script, '--check', '--mutant', m], cwd=root, capture_output=True, timeout=600).returncode - if rc != 1: - raise RuntimeError('mutant ' + m + ' not rejected under ' + ' '.join(flags)) + from tools.check_mode_replay import replay + replay('c8-cap-failure-coefficient-dims', root, timeout=600) print(json.dumps({'scientific_effect': 'NONE', 'certified_values_replayed': True, 'mutants_rejected': 8})) PY git diff --exit-code diff --git a/.github/workflows/c8-normalizer-floor-planar.yml b/.github/workflows/c8-normalizer-floor-planar.yml index c3eeb1758..68ceb7005 100644 --- a/.github/workflows/c8-normalizer-floor-planar.yml +++ b/.github/workflows/c8-normalizer-floor-planar.yml @@ -5,6 +5,10 @@ name: C8 planar normalizer floor (certified replay) paths: - 'frontiers/c8_normalizer_floor_planar_20260930/**' - '.github/workflows/c8-normalizer-floor-planar.yml' + - 'tools/**' + - 'tools.py' + - 'tests/test_check_mode_workflows.py' + - 'tests/test_legacy_workflow_triggers.py' - 'imports/lifetime_parent_20260925/UNIFORM_MATRIX_CAP_AND_LIFETIME.md' - 'frontiers/c6_cluster_coefficients_numerics_20260930/RESULTS.json' workflow_dispatch: @@ -23,6 +27,13 @@ jobs: - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 with: python-version: '3.11.16' + - name: Check trigger coverage and the complete shell rejection contract in both modes + env: + CHECK_MODE_TEST_FAMILY: 'c8-normalizer-floor-planar' + PYTHONDONTWRITEBYTECODE: '1' + run: | + python -B -S tests/test_check_mode_workflows.py + python -B -O -S tests/test_check_mode_workflows.py - name: Verify manifest and main-resident pins, replay the certified floor (twelve sub-boxes of the two bands exactly), reject mutants env: PYTHONDONTWRITEBYTECODE: '1' @@ -50,15 +61,8 @@ jobs: blob = hashlib.sha1(b'blob %d\0' % len(data) + data).hexdigest() if len(data) != entry['bytes'] or hashlib.sha256(data).hexdigest() != entry['sha256'] or blob != entry['git_blob']: raise ValueError('consumed source identity mismatch on this tree: ' + entry['path']) - script = str(root / 'floor.py') - for flags in (['-B', '-S'], ['-B', '-O', '-S']): - out = subprocess.run([sys.executable, *flags, script, '--check'], cwd=root, capture_output=True, timeout=1500) - if out.returncode != 0: - raise RuntimeError('check failed under ' + ' '.join(flags) + ': ' + out.stdout.decode()[-400:] + out.stderr.decode()[-400:]) - for m in ['swap-minmax', 'drop-torus-error', 'sign-saddle']: - rc = subprocess.run([sys.executable, *flags, script, '--check', '--mutant', m], cwd=root, capture_output=True, timeout=1500).returncode - if rc != 1: - raise RuntimeError('mutant ' + m + ' not rejected') + from tools.check_mode_replay import replay + replay('c8-normalizer-floor-planar', root, timeout=1500) print(json.dumps({'scientific_effect': 'NONE', 'certified_floor_replayed': True})) PY git diff --exit-code diff --git a/reviews/c8_check_mode_wrappers_20261026/SCOPE.md b/reviews/c8_check_mode_wrappers_20261026/SCOPE.md new file mode 100644 index 000000000..97acd7db5 --- /dev/null +++ b/reviews/c8_check_mode_wrappers_20261026/SCOPE.md @@ -0,0 +1,143 @@ +# C8 check-mode wrappers, batch 1: authenticate the intended verdicts + +Dylan Roy — delegated AI engineering. Actual author: Anthropic Claude, session +`session_015wNj8LPTKXsaT68G3DgPPh` (C8), stage `C8-WRAPPERS-REPAIR-C8-20261026`, +pickup Math-#322/6422210000. Dylan approved draft PRs for the seven `c8-*` +wrappers around this session's packets, in the author's session at about 16:45Z +on 26 October 2026. Scientific effect NONE; shared-account organizational credit 0. +Author-side implementation only: a separate nonauthor engineering review and a +separate integrator are required. No mathematical acceptance is asserted. + +The branch is cut from Math- `main` `1965142e`, the merge of #518. The +repository-wide suite that runs these new tests needs #518's 45-minute budget. + +## Finding and exact source + +01Cz's ledger (Math-#322/6388271177) classifies both wrappers as `status_only`. +Each mutant was accepted on exit status 1 alone, with its stdout and stderr +discarded. The baseline `--check` was accepted on exit status 0 alone, with its +stderr discarded and its stdout only echoed. So an unrelated crash, a silent exit +1 or a failure for another reason counted as an intended rejection. + +| Object | Identity | +|---|---| +| Math- `main` at characterization | `f0f90857dcf7586e5195fcb05c0058657fba4003` | +| Branch base (`main`, the merge of #518) | `1965142e66a067c69ea2b936517226a6bbf2cdd8` | +| `c8-cap-failure-coefficient-dims.yml` before | blob `afb65fea440ffde64731d5d16bc3aafb7aa2d250` | +| `c8-normalizer-floor-planar.yml` before | blob `c3eeb1758cee919bac592db55c190bb76ddfd4e1` | +| `cap_coefficient_dims.py` (unchanged) | blob `af7e9d103f962f613c57f5e695f030165e348beb`, 24,393 B | +| `floor.py` (unchanged) | blob `29f1b512062fda6b3565429e9fe53e91b9b3c2f8`, 44,876 B | +| dims `RESULTS.json` (unchanged) | 6,837 B, SHA-256 `1406d5138ac3d33bcb0358f6a3005a67e3a4a2dbfc2c43bc3ac2593667550608` | +| normalizer `RESULTS.json` (unchanged) | 1,874,499 B, SHA-256 `a2c4d6890cc9ab0984605f78ccf89b38d93b9515b8d70a6d5e5c3d417b4b3882` | + +Both packet trees, `4866eab3` (dims) and `f734b522` (normalizer), are identical at +`f0f90857` and at `1965142e`, and so are both workflow blobs. The imported +`tools/legacy_json_replay.py` (blob `854db466`) and +`tests/test_legacy_workflow_triggers.py` (blob `c2e37a3a`) are the blobs every +local run below used. + +## Repair and preservation + +Six files: `tools/check_mode_replay.py`, `tools/check_mode_contracts.json` and +`tests/test_check_mode_workflows.py` (all new), the two workflows, and this note. + +`tools/check_mode_replay.py` follows `tools/legacy_json_replay.py`, and it reuses +that module's strict JSON reader and structural comparison instead of copying +them. For each family it binds the checker's git blob and the `RESULTS.json` size +and digest. It then runs the fixed inventory in both modes, from the packet +directory, exactly as the old wrappers did: `--check` and `--check --mutant M`. +For every child it requires: + +- the exact exit status: 0 for the baseline, 1 for each mutant; +- empty stderr; +- one strict JSON verdict on stdout, structurally equal to the verdict measured + from the unchanged checker. + +Each workflow now calls `replay()` in place of its own exit-only loops. It gains +one earlier step that runs `tests/test_check_mode_workflows.py` for its own family +in both modes. It also gains the trigger paths for the helper and its inputs +(`tools/**`, `tools.py`), for that test, and for `tests/test_legacy_workflow_triggers.py`, +whose strict path reader the test reuses. + +Unchanged: +- the source block before any child: the packet manifest and the main-resident + pins; +- the action pins, read-only permissions, `persist-credentials: false`, Python + 3.11.16 and the job budgets; +- the per-child timeouts (600 s and 1500 s), the mutant order within each mode + and the terminal record; +- the final `git diff --exit-code`; +- the normalizer's `workflow_dispatch` full-regeneration job; +- every packet file. + +One ordering detail changes, in the dims wrapper only. It ran both baselines +before any mutant. The helper runs each mode's complete inventory in turn, as the +normalizer wrapper already did, and as the characterization runs measured it. +Each child is still a separate process started from the packet directory. + +## The contract + +The references are complete verdicts measured from the unchanged CLIs, never +outputs learned from the invocation under test. + +- **Baseline:** exit 0, empty stderr, and the measured `{"check": "ok", …, + "mutant": null}` verdict, which carries the certified values that `--check` has + already compared with `RESULTS.json`. +- **Each mutant:** exit 1, empty stderr, and exactly the verdict that the checker's + `require()` prints: `{"check": "FAIL", "reason": …, "mutant": M}`. + +| Family | Mutants (in run order) | Measured reason | +|---|---|---| +| `c8-cap-failure-coefficient-dims` | `no-y-tail`, `r3-one`, `no-bp`, `frob-omega-off` | `certified values differ from RESULTS.json` | +| `c8-normalizer-floor-planar` | `swap-minmax`, `drop-torus-error`, `sign-saddle` | `exact replay of low r=[63/512,1/8],b=[0,1/8],k=[1/2,3/4]` | + +Within each family every mutant fails the same first check. The verdict names +that check, and the mutant label, but not which quantity the mutant changed. That +is how these checkers report, and the packet bytes stay unchanged. The contract +therefore pins the first failing check exactly. A crash, a silent exit, stderr +output, a failure at any other check, or a verdict for another label is rejected. + +## Execution and non-vacuous regressions + +All of the following ran locally on CPython 3.11.15. The characterization and the +real-checker replay ran on #518's head `b0dd89f`. The tests and the RED control ran +again on `1965142e`. + +- **Characterization:** two separate runs of the unchanged checkers, in both + modes. Every normal and optimized stdout is byte-identical, every stderr is + empty, and the exit statuses are as above. Locally, the normalizer baseline + takes about 6 minutes per mode, each normalizer mutant 28 to 33 seconds, and + each dims child under half a second. +- **Real checkers through the helper:** `replay()` on both actual packets gave + `protocol_ok` for every child: 10 for dims in 2.5 seconds and 8 for the + normalizer in about 14.5 minutes. Every stdout digest equals the characterized + one. +- **This test file:** 13 tests OK in both modes, both as the workflow step runs it + and under the full suite's discovery, about 23 seconds per mode for both + families. +- **RED control:** the same tests, with `CHECK_MODE_SOURCE_ROOT` set to unchanged + `main`, give 90 false acceptances: + + | Test | dims | normalizer | + |---|---|---| + | crashes | 8 | 6 | + | silent exits | 8 | 6 | + | wrong reasons | 8 | 6 | + | wrong mutant label | 2 | 2 | + | baseline faults (stderr, wrong value, missing key, malformed) | 8 | 8 | + | JSON faults (stderr, duplicate, NaN, trailing, extra, missing, malformed) | 14 | 14 | + + The wrong exit statuses were already rejected there and still are. The trigger, + step and import tests fail there too, as expected, and so does the dims + inventory-order check, because the old dims wrapper ran both baselines first. + +## Limits + +- The local interpreter is 3.11.15, not the hosted 3.11.16. The hosted runs are + the qualifying execution. +- The synthetic fixture uses a one-entry pin list and a minimal manifest. The real + pins run only in the real hosted job. +- This is batch 1 of seven. `c8-cap-failure-coefficient-planar`, + `c8-lifetime-difference-constant`, `c8-theorem-a-constants-d3`, + `c8-theorem-a-constants-planar` and `c8-torus-vanishing-remainder` keep their + own stages. diff --git a/tests/test_check_mode_workflows.py b/tests/test_check_mode_workflows.py new file mode 100644 index 000000000..11ec86696 --- /dev/null +++ b/tests/test_check_mode_workflows.py @@ -0,0 +1,294 @@ +"""Synthetic whole-shell regressions for the C8 check-mode wrappers. + +The fixture commits synthetic source, manifest and upstream bytes, then runs the +actual verification step of each wrapper with real subprocesses and the measured +verdicts from tools/check_mode_contracts.json. It also checks each wrapper's +trigger paths, step order and import line. This is protocol coverage, not a +field calculation. +""" +from __future__ import annotations +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import textwrap +import unittest + +from test_legacy_workflow_triggers import covered, trigger_paths + +ROOT = Path(__file__).resolve().parents[1] +SELF = 'tests/test_check_mode_workflows.py' +SOURCE = Path(os.environ.get('CHECK_MODE_SOURCE_ROOT', str(ROOT))) +CONTRACTS = json.loads((ROOT / 'tools/check_mode_contracts.json').read_text()) +WORKFLOWS = { + 'c8-cap-failure-coefficient-dims': 'c8-cap-failure-coefficient-dims.yml', + 'c8-normalizer-floor-planar': 'c8-normalizer-floor-planar.yml', +} +ACTIVE = {k: v for k, v in CONTRACTS['families'].items() + if not os.environ.get('CHECK_MODE_TEST_FAMILY') or k == os.environ['CHECK_MODE_TEST_FAMILY']} +if not ACTIVE: + raise SystemExit('unknown test family') +# Mutant order within each mode, as in each wrapper's original loop. +ORDERS = { + 'c8-cap-failure-coefficient-dims': ('no-y-tail', 'r3-one', 'no-bp', 'frob-omega-off'), + 'c8-normalizer-floor-planar': ('swap-minmax', 'drop-torus-error', 'sign-saddle'), +} +MODES = ('normal', 'optimized') +VERIFY = ' - name: Verify manifest and main-resident pins' +STEP = ( + ' - name: Check trigger coverage and the complete shell rejection contract in both modes\n' + ' env:\n' + " CHECK_MODE_TEST_FAMILY: '%s'\n" + " PYTHONDONTWRITEBYTECODE: '1'\n" + ' run: |\n' + ' python -B -S ' + SELF + '\n' + ' python -B -O -S ' + SELF + '\n' +) + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def blob(data): + return hashlib.sha1(b'blob %d\0' % len(data) + data).hexdigest() + + +def row(path, data): + return {'path': path, 'bytes': len(data), 'sha256': digest(data), 'git_blob': blob(data)} + + +def workflow_text(family, source=SOURCE): + return (source / '.github/workflows' / WORKFLOWS[family]).read_text(encoding='utf-8') + + +def verification_shell(text): + lines = text.splitlines() + start = next(i for i, line in enumerate(lines) if line.startswith(VERIFY)) + start = next(i for i in range(start, len(lines)) if lines[i].strip() == 'run: |') + 1 + end = start + while end < len(lines) and (not lines[end].strip() or lines[end].startswith(' ')): + end += 1 + return textwrap.dedent('\n'.join(lines[start:end])) + '\n' + + +CHILD = r''' +import json, os, sys +baseline = BASELINE +reports = REPORTS +args = sys.argv[1:] +if args[:1] != ['--check'] or len(args) not in (1, 3) or (len(args) == 3 and args[1] != '--mutant'): + sys.stderr.write('unexpected arguments\n') + sys.exit(3) +label = args[2] if len(args) == 3 else 'baseline' +mode = 'optimized' if sys.flags.optimize else 'normal' +with open(os.environ['CHECK_MODE_TRACE'], 'a') as f: + f.write(json.dumps([label, mode, os.getcwd()]) + '\n') +value, rc = (baseline, 0) if label == 'baseline' else (reports[label], 1) +output = json.dumps(value) + '\n' +if os.environ.get('CHECK_MODE_TARGET') == label and os.environ.get('CHECK_MODE_MODE') == mode: + kind = os.environ.get('CHECK_MODE_FAULT', '') + if kind == 'crash': + raise RuntimeError('unrelated fixture crash') + if kind == 'silent': + sys.exit(rc) + if kind == 'stderr': + sys.stderr.write('unexpected diagnostic\n') + if kind == 'wrong-exit': + rc = 0 if label != 'baseline' else 1 + if kind == 'wrong-reason': + output = json.dumps(dict(value, reason='unrelated check failed')) + '\n' + if kind == 'wrong-label': + output = json.dumps(dict(value, mutant='another-mutant')) + '\n' + if kind == 'wrong-value': + name = next(k for k in value if k not in ('check', 'mutant')) + output = json.dumps(dict(value, **{name: 'unrelated value'})) + '\n' + if kind == 'format': + output = ' \n' + json.dumps(value, indent=4, sort_keys=True) + '\n\t' + if kind == 'duplicate': + output = output.rstrip()[:-1] + ', "check": ' + json.dumps(value['check']) + '}\n' + if kind == 'nonfinite': + output = output.rstrip()[:-1] + ', "unexpected": NaN}\n' + if kind == 'trailing': + output += '{}\n' + if kind == 'extra': + output = json.dumps(dict(value, unexpected='x')) + '\n' + if kind == 'missing': + output = json.dumps({k: v for k, v in value.items() if k != 'check'}) + '\n' + if kind == 'malformed': + output = 'not JSON\n' +sys.stdout.write(output) +sys.exit(rc) +''' + + +class CheckModeWorkflowTests(unittest.TestCase): + maxDiff = None + + def case(self, family, label='baseline', mode='normal', fault='', tamper=None): + with tempfile.TemporaryDirectory(prefix='check-mode-test-') as td: + outside = Path(td) + repo = outside / 'repo' + repo.mkdir() + trace = outside / 'trace' + contract = json.loads(json.dumps(CONTRACTS)) + entry = contract['families'][family] + directory = repo / entry['directory'] + directory.mkdir(parents=True) + script = CHILD.replace('BASELINE', repr(entry['baseline'])).replace('REPORTS', repr(entry['mutants'])) + data = script.encode() + (directory / entry['script']).write_bytes(data) + results = (ROOT / entry['directory'] / 'RESULTS.json').read_bytes() + (directory / 'RESULTS.json').write_bytes(results) + upstream = b'synthetic upstream; no scientific authority\n' + (repo / 'upstream.txt').write_bytes(upstream) + pin = dict(row('upstream.txt', upstream), current_required=True) + source_map = json.dumps({'sources': [pin]}).encode() + (directory / 'SOURCE_MAP.json').write_bytes(source_map) + files = [row(entry['script'], data), row('RESULTS.json', results), row('SOURCE_MAP.json', source_map)] + (directory / 'SOURCE_FILES.json').write_text(json.dumps({'files': files})) + (repo / 'tools').mkdir() + for helper in ('check_mode_replay.py', 'legacy_json_replay.py'): + if (SOURCE / 'tools' / helper).exists(): + shutil.copyfile(SOURCE / 'tools' / helper, repo / 'tools' / helper) + entry['script_git_blob'] = blob(data) + # Only the checker identity is re-bound for this synthetic fixture. + (repo / 'tools/check_mode_contracts.json').write_text(json.dumps(contract)) + env = os.environ.copy() + env.update({'CHECK_MODE_TRACE': str(trace), 'CHECK_MODE_TARGET': label, 'CHECK_MODE_MODE': mode, + 'CHECK_MODE_FAULT': fault, 'PYTHONDONTWRITEBYTECODE': '1'}) + for cmd in (['git', 'init', '-q'], ['git', 'add', '.'], + ['git', '-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', + '-c', 'commit.gpgsign=false', 'commit', '-qm', 'synthetic fixture']): + subprocess.run(cmd, cwd=repo, check=True, capture_output=True, timeout=15, env=env) + if tamper == 'packet': + (directory / entry['script']).write_bytes(data + b'\n') + if tamper == 'upstream': + (repo / 'upstream.txt').write_bytes(upstream + b'x') + shell = verification_shell(workflow_text(family)) + result = subprocess.run(['bash', '-c', shell], cwd=repo, env=env, capture_output=True, timeout=90) + events = [json.loads(s) for s in trace.read_text().splitlines()] if trace.exists() else [] + return result, events, str(directory.resolve()) + + def rejected(self, family, label, mode, fault): + result, events, _ = self.case(family, label, mode, fault) + self.assertNotEqual(result.returncode, 0, (family, label, mode, fault, 'false acceptance', events)) + self.assertEqual(events[-1][:2], [label, mode], + (family, label, mode, fault, events, result.stderr.decode(errors='replace'))) + + def test_reference_custody(self): + canonical = json.dumps(CONTRACTS['families'], sort_keys=True, separators=(',', ':')).encode() + self.assertEqual(digest(canonical), '9127eaa7bc8b14bf1187940fa89c0e59ad94015420a82d291147e83f3bafb2e9') + for family, entry in ACTIVE.items(): + with self.subTest(family=family): + packet = ROOT / entry['directory'] + self.assertEqual(blob((packet / entry['script']).read_bytes()), entry['script_git_blob']) + results = (packet / 'RESULTS.json').read_bytes() + self.assertEqual((len(results), digest(results)), (entry['results_bytes'], entry['results_sha256'])) + + def test_valid_complete_inventory(self): + for family in ACTIVE: + with self.subTest(family=family): + result, events, packet = self.case(family) + self.assertEqual(result.returncode, 0, result.stderr.decode(errors='replace')) + labels = ['baseline', *ORDERS[family]] + self.assertEqual(events, [[label, mode, packet] for mode in MODES for label in labels]) + + def test_formatting_alone_is_accepted(self): + for family in ACTIVE: + for label in ('baseline', ORDERS[family][0]): + with self.subTest(family=family, label=label): + result, _, _ = self.case(family, label, 'optimized', 'format') + self.assertEqual(result.returncode, 0, result.stderr.decode(errors='replace')) + + def test_crashes_each_mutant_each_mode(self): + for family in ACTIVE: + for label in ORDERS[family]: + for mode in MODES: + with self.subTest(family=family, label=label, mode=mode): + self.rejected(family, label, mode, 'crash') + + def test_silent_exits_each_mutant_each_mode(self): + for family in ACTIVE: + for label in ORDERS[family]: + for mode in MODES: + with self.subTest(family=family, label=label, mode=mode): + self.rejected(family, label, mode, 'silent') + + def test_wrong_reasons_each_mutant_each_mode(self): + for family in ACTIVE: + for label in ORDERS[family]: + for mode in MODES: + with self.subTest(family=family, label=label, mode=mode): + self.rejected(family, label, mode, 'wrong-reason') + + def test_wrong_mutant_label(self): + for family in ACTIVE: + for mode in MODES: + with self.subTest(family=family, mode=mode): + self.rejected(family, ORDERS[family][0], mode, 'wrong-label') + + def test_baseline_contract(self): + for family in ACTIVE: + for fault in ('stderr', 'wrong-value', 'wrong-exit', 'missing', 'malformed'): + for mode in MODES: + with self.subTest(family=family, fault=fault, mode=mode): + self.rejected(family, 'baseline', mode, fault) + + def test_json_contract(self): + for family in ACTIVE: + label = ORDERS[family][0] + for fault in ('stderr', 'wrong-exit', 'duplicate', 'nonfinite', 'trailing', 'extra', 'missing', + 'malformed'): + for mode in MODES: + with self.subTest(family=family, fault=fault, mode=mode): + self.rejected(family, label, mode, fault) + + def test_source_guards_run_before_any_child(self): + for family in ACTIVE: + for tamper in ('packet', 'upstream'): + with self.subTest(family=family, tamper=tamper): + result, events, _ = self.case(family, tamper=tamper) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(events, []) + + +class CheckModeTriggerTests(unittest.TestCase): + def test_paths_cover_the_helper_its_inputs_and_these_tests(self): + for family, entry in ACTIVE.items(): + paths = trigger_paths(workflow_text(family)) + for path in (entry['directory'] + '/' + entry['script'], '.github/workflows/' + WORKFLOWS[family], + 'tools/check_mode_replay.py', 'tools/check_mode_contracts.json', + 'tools/legacy_json_replay.py', 'tools/__init__.py', 'tools.py', SELF, + 'tests/test_legacy_workflow_triggers.py'): + with self.subTest(family=family, path=path): + self.assertTrue(covered(paths, path), 'consumed or regression path is uncovered') + for path in ('README.md', 'unrelated/tools.py', 'tools_extra/file.py'): + with self.subTest(family=family, unrelated=path): + self.assertFalse(covered(paths, path), 'unexpected unrelated trigger') + + def test_contract_step_runs_in_both_modes_before_the_replay(self): + for family in ACTIVE: + text = workflow_text(family) + with self.subTest(family=family): + step = STEP % family + self.assertEqual(text.count(step), 1, 'missing exact two-mode contract step') + self.assertLess(text.index(step), text.index(VERIFY)) + + def test_the_replay_is_the_only_import_and_names_its_family(self): + for family in ACTIVE: + text = workflow_text(family) + with self.subTest(family=family): + lines = [s.strip() for s in text.splitlines() if s.strip().startswith('from tools.')] + self.assertEqual(lines, ['from tools.check_mode_replay import replay']) + self.assertEqual(text.count("replay('%s', root" % family), 1) + # No exit-status-only check of a child remains beside the replay. + self.assertNotIn('.returncode', verification_shell(text)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/check_mode_contracts.json b/tools/check_mode_contracts.json new file mode 100644 index 000000000..2c25b879b --- /dev/null +++ b/tools/check_mode_contracts.json @@ -0,0 +1 @@ +{"families":{"c8-cap-failure-coefficient-dims":{"baseline":{"cG3_0":{"1":["2803036.717892","2803338.074669"],"1/2":["358497.545122","619756.489477"],"2":["22424320.655069","22424320.655072"],"3/2":["9460258.333462","9460258.334033"],"3/4":["1182683.064519","1201290.889423"]},"check":"ok","mutant":null},"directory":"frontiers/c8_cap_failure_coefficient_dims_20261001","mutants":{"frob-omega-off":{"check":"FAIL","mutant":"frob-omega-off","reason":"certified values differ from RESULTS.json"},"no-bp":{"check":"FAIL","mutant":"no-bp","reason":"certified values differ from RESULTS.json"},"no-y-tail":{"check":"FAIL","mutant":"no-y-tail","reason":"certified values differ from RESULTS.json"},"r3-one":{"check":"FAIL","mutant":"r3-one","reason":"certified values differ from RESULTS.json"}},"results_bytes":6837,"results_sha256":"1406d5138ac3d33bcb0358f6a3005a67e3a4a2dbfc2c43bc3ac2593667550608","script":"cap_coefficient_dims.py","script_git_blob":"af7e9d103f962f613c57f5e695f030165e348beb"},"c8-normalizer-floor-planar":{"baseline":{"check":"ok","mutant":null,"z_star_combined":"3.5044542729","z_star_high":"3.5044542729","z_star_low":"7.0499921959"},"directory":"frontiers/c8_normalizer_floor_planar_20260930","mutants":{"drop-torus-error":{"check":"FAIL","mutant":"drop-torus-error","reason":"exact replay of low r=[63/512,1/8],b=[0,1/8],k=[1/2,3/4]"},"sign-saddle":{"check":"FAIL","mutant":"sign-saddle","reason":"exact replay of low r=[63/512,1/8],b=[0,1/8],k=[1/2,3/4]"},"swap-minmax":{"check":"FAIL","mutant":"swap-minmax","reason":"exact replay of low r=[63/512,1/8],b=[0,1/8],k=[1/2,3/4]"}},"results_bytes":1874499,"results_sha256":"a2c4d6890cc9ab0984605f78ccf89b38d93b9515b8d70a6d5e5c3d417b4b3882","script":"floor.py","script_git_blob":"29f1b512062fda6b3565429e9fe53e91b9b3c2f8"}},"provenance":"Native unchanged check-mode verdicts of the C8 checkers, measured for Math-#322 (C8-WRAPPERS-REPAIR-C8-20261026) in both modes from each packet directory. This file is an engineering reference, not mathematical acceptance.","schema":1,"source_cut":"f0f90857dcf7586e5195fcb05c0058657fba4003"} diff --git a/tools/check_mode_replay.py b/tools/check_mode_replay.py new file mode 100644 index 000000000..59f0b8c5f --- /dev/null +++ b/tools/check_mode_replay.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Authenticate the C8 check-mode checkers' acceptance and rejection verdicts. + +Called AFTER each workflow's unchanged source and pin guards. Each checker runs +from its packet directory as `