From c168438b4ff34afafb4726a3c8f1e43c514164cc Mon Sep 17 00:00:00 2001 From: Dylan Roy Date: Mon, 26 Oct 2026 18:22:22 +0000 Subject: [PATCH] Authenticate the C8 check-mode verdicts for two wrappers The c8-cap-failure-coefficient-dims and c8-normalizer-floor-planar workflows accepted each mutant on exit status 1 alone and the baseline on exit status 0 alone, so an unrelated crash, a silent exit or a failure for another reason counted as an intended rejection. Both now call tools/check_mode_replay.py. It binds the checker blob and the RESULTS.json identity, then requires, for every child in both interpreter modes, the exact exit status, empty stderr and the complete verdict measured from the unchanged checker. It reuses the strict JSON reader and structural comparison of tools/legacy_json_replay.py. tests/test_check_mode_workflows.py runs each workflow's real verification shell against synthetic faults, and each workflow runs it for its own family before the replay. Packet files are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015wNj8LPTKXsaT68G3DgPPh --- .../c8-cap-failure-coefficient-dims.yml | 24 +- .../workflows/c8-normalizer-floor-planar.yml | 22 +- .../c8_check_mode_wrappers_20261026/SCOPE.md | 143 +++++++++ tests/test_check_mode_workflows.py | 294 ++++++++++++++++++ tools/check_mode_contracts.json | 1 + tools/check_mode_replay.py | 115 +++++++ 6 files changed, 579 insertions(+), 20 deletions(-) create mode 100644 reviews/c8_check_mode_wrappers_20261026/SCOPE.md create mode 100644 tests/test_check_mode_workflows.py create mode 100644 tools/check_mode_contracts.json create mode 100644 tools/check_mode_replay.py diff --git a/.github/workflows/c8-cap-failure-coefficient-dims.yml b/.github/workflows/c8-cap-failure-coefficient-dims.yml index afb65fea4..fe18ccae5 100644 --- a/.github/workflows/c8-cap-failure-coefficient-dims.yml +++ b/.github/workflows/c8-cap-failure-coefficient-dims.yml @@ -5,6 +5,10 @@ name: C8 cap-failure coefficient in every dimension (certified replay) paths: - 'frontiers/c8_cap_failure_coefficient_dims_20261001/**' - '.github/workflows/c8-cap-failure-coefficient-dims.yml' + - 'tools/**' + - 'tools.py' + - 'tests/test_check_mode_workflows.py' + - 'tests/test_legacy_workflow_triggers.py' - 'imports/lifetime_parent_20260925/UNIFORM_MATRIX_CAP_AND_LIFETIME.md' - 'imports/lifetime_parent_20260925/MARKED_CYLINDER_CAP_PROOF.md' - 'frontiers/spectral_cluster_closure_20260929/PROOF.md' @@ -24,6 +28,13 @@ jobs: - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 with: python-version: '3.11.16' + - name: Check trigger coverage and the complete shell rejection contract in both modes + env: + CHECK_MODE_TEST_FAMILY: 'c8-cap-failure-coefficient-dims' + PYTHONDONTWRITEBYTECODE: '1' + run: | + python -B -S tests/test_check_mode_workflows.py + python -B -O -S tests/test_check_mode_workflows.py - name: Verify manifest and main-resident pins, replay the certified values in both interpreter modes, reject mutants env: PYTHONDONTWRITEBYTECODE: '1' @@ -51,17 +62,8 @@ jobs: blob = hashlib.sha1(b'blob %d\0' % len(data) + data).hexdigest() if len(data) != entry['bytes'] or hashlib.sha256(data).hexdigest() != entry['sha256'] or blob != entry['git_blob']: raise ValueError('consumed source identity mismatch on this tree: ' + entry['path']) - script = str(root / 'cap_coefficient_dims.py') - for flags in (['-B', '-S'], ['-B', '-O', '-S']): - out = subprocess.run([sys.executable, *flags, script, '--check'], cwd=root, capture_output=True, timeout=600) - if out.returncode != 0: - raise RuntimeError('check failed under ' + ' '.join(flags) + ': ' + out.stdout.decode()[-400:] + out.stderr.decode()[-400:]) - print(flags, out.stdout.decode().strip()[-300:]) - for flags in (['-B', '-S'], ['-B', '-O', '-S']): - for m in ('no-y-tail', 'r3-one', 'no-bp', 'frob-omega-off'): - rc = subprocess.run([sys.executable, *flags, script, '--check', '--mutant', m], cwd=root, capture_output=True, timeout=600).returncode - if rc != 1: - raise RuntimeError('mutant ' + m + ' not rejected under ' + ' '.join(flags)) + from tools.check_mode_replay import replay + replay('c8-cap-failure-coefficient-dims', root, timeout=600) print(json.dumps({'scientific_effect': 'NONE', 'certified_values_replayed': True, 'mutants_rejected': 8})) PY git diff --exit-code diff --git a/.github/workflows/c8-normalizer-floor-planar.yml b/.github/workflows/c8-normalizer-floor-planar.yml index c3eeb1758..68ceb7005 100644 --- a/.github/workflows/c8-normalizer-floor-planar.yml +++ b/.github/workflows/c8-normalizer-floor-planar.yml @@ -5,6 +5,10 @@ name: C8 planar normalizer floor (certified replay) paths: - 'frontiers/c8_normalizer_floor_planar_20260930/**' - '.github/workflows/c8-normalizer-floor-planar.yml' + - 'tools/**' + - 'tools.py' + - 'tests/test_check_mode_workflows.py' + - 'tests/test_legacy_workflow_triggers.py' - 'imports/lifetime_parent_20260925/UNIFORM_MATRIX_CAP_AND_LIFETIME.md' - 'frontiers/c6_cluster_coefficients_numerics_20260930/RESULTS.json' workflow_dispatch: @@ -23,6 +27,13 @@ jobs: - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 with: python-version: '3.11.16' + - name: Check trigger coverage and the complete shell rejection contract in both modes + env: + CHECK_MODE_TEST_FAMILY: 'c8-normalizer-floor-planar' + PYTHONDONTWRITEBYTECODE: '1' + run: | + python -B -S tests/test_check_mode_workflows.py + python -B -O -S tests/test_check_mode_workflows.py - name: Verify manifest and main-resident pins, replay the certified floor (twelve sub-boxes of the two bands exactly), reject mutants env: PYTHONDONTWRITEBYTECODE: '1' @@ -50,15 +61,8 @@ jobs: blob = hashlib.sha1(b'blob %d\0' % len(data) + data).hexdigest() if len(data) != entry['bytes'] or hashlib.sha256(data).hexdigest() != entry['sha256'] or blob != entry['git_blob']: raise ValueError('consumed source identity mismatch on this tree: ' + entry['path']) - script = str(root / 'floor.py') - for flags in (['-B', '-S'], ['-B', '-O', '-S']): - out = subprocess.run([sys.executable, *flags, script, '--check'], cwd=root, capture_output=True, timeout=1500) - if out.returncode != 0: - raise RuntimeError('check failed under ' + ' '.join(flags) + ': ' + out.stdout.decode()[-400:] + out.stderr.decode()[-400:]) - for m in ['swap-minmax', 'drop-torus-error', 'sign-saddle']: - rc = subprocess.run([sys.executable, *flags, script, '--check', '--mutant', m], cwd=root, capture_output=True, timeout=1500).returncode - if rc != 1: - raise RuntimeError('mutant ' + m + ' not rejected') + from tools.check_mode_replay import replay + replay('c8-normalizer-floor-planar', root, timeout=1500) print(json.dumps({'scientific_effect': 'NONE', 'certified_floor_replayed': True})) PY git diff --exit-code diff --git a/reviews/c8_check_mode_wrappers_20261026/SCOPE.md b/reviews/c8_check_mode_wrappers_20261026/SCOPE.md new file mode 100644 index 000000000..97acd7db5 --- /dev/null +++ b/reviews/c8_check_mode_wrappers_20261026/SCOPE.md @@ -0,0 +1,143 @@ +# C8 check-mode wrappers, batch 1: authenticate the intended verdicts + +Dylan Roy — delegated AI engineering. Actual author: Anthropic Claude, session +`session_015wNj8LPTKXsaT68G3DgPPh` (C8), stage `C8-WRAPPERS-REPAIR-C8-20261026`, +pickup Math-#322/6422210000. Dylan approved draft PRs for the seven `c8-*` +wrappers around this session's packets, in the author's session at about 16:45Z +on 26 October 2026. Scientific effect NONE; shared-account organizational credit 0. +Author-side implementation only: a separate nonauthor engineering review and a +separate integrator are required. No mathematical acceptance is asserted. + +The branch is cut from Math- `main` `1965142e`, the merge of #518. The +repository-wide suite that runs these new tests needs #518's 45-minute budget. + +## Finding and exact source + +01Cz's ledger (Math-#322/6388271177) classifies both wrappers as `status_only`. +Each mutant was accepted on exit status 1 alone, with its stdout and stderr +discarded. The baseline `--check` was accepted on exit status 0 alone, with its +stderr discarded and its stdout only echoed. So an unrelated crash, a silent exit +1 or a failure for another reason counted as an intended rejection. + +| Object | Identity | +|---|---| +| Math- `main` at characterization | `f0f90857dcf7586e5195fcb05c0058657fba4003` | +| Branch base (`main`, the merge of #518) | `1965142e66a067c69ea2b936517226a6bbf2cdd8` | +| `c8-cap-failure-coefficient-dims.yml` before | blob `afb65fea440ffde64731d5d16bc3aafb7aa2d250` | +| `c8-normalizer-floor-planar.yml` before | blob `c3eeb1758cee919bac592db55c190bb76ddfd4e1` | +| `cap_coefficient_dims.py` (unchanged) | blob `af7e9d103f962f613c57f5e695f030165e348beb`, 24,393 B | +| `floor.py` (unchanged) | blob `29f1b512062fda6b3565429e9fe53e91b9b3c2f8`, 44,876 B | +| dims `RESULTS.json` (unchanged) | 6,837 B, SHA-256 `1406d5138ac3d33bcb0358f6a3005a67e3a4a2dbfc2c43bc3ac2593667550608` | +| normalizer `RESULTS.json` (unchanged) | 1,874,499 B, SHA-256 `a2c4d6890cc9ab0984605f78ccf89b38d93b9515b8d70a6d5e5c3d417b4b3882` | + +Both packet trees, `4866eab3` (dims) and `f734b522` (normalizer), are identical at +`f0f90857` and at `1965142e`, and so are both workflow blobs. The imported +`tools/legacy_json_replay.py` (blob `854db466`) and +`tests/test_legacy_workflow_triggers.py` (blob `c2e37a3a`) are the blobs every +local run below used. + +## Repair and preservation + +Six files: `tools/check_mode_replay.py`, `tools/check_mode_contracts.json` and +`tests/test_check_mode_workflows.py` (all new), the two workflows, and this note. + +`tools/check_mode_replay.py` follows `tools/legacy_json_replay.py`, and it reuses +that module's strict JSON reader and structural comparison instead of copying +them. For each family it binds the checker's git blob and the `RESULTS.json` size +and digest. It then runs the fixed inventory in both modes, from the packet +directory, exactly as the old wrappers did: `--check` and `--check --mutant M`. +For every child it requires: + +- the exact exit status: 0 for the baseline, 1 for each mutant; +- empty stderr; +- one strict JSON verdict on stdout, structurally equal to the verdict measured + from the unchanged checker. + +Each workflow now calls `replay()` in place of its own exit-only loops. It gains +one earlier step that runs `tests/test_check_mode_workflows.py` for its own family +in both modes. It also gains the trigger paths for the helper and its inputs +(`tools/**`, `tools.py`), for that test, and for `tests/test_legacy_workflow_triggers.py`, +whose strict path reader the test reuses. + +Unchanged: +- the source block before any child: the packet manifest and the main-resident + pins; +- the action pins, read-only permissions, `persist-credentials: false`, Python + 3.11.16 and the job budgets; +- the per-child timeouts (600 s and 1500 s), the mutant order within each mode + and the terminal record; +- the final `git diff --exit-code`; +- the normalizer's `workflow_dispatch` full-regeneration job; +- every packet file. + +One ordering detail changes, in the dims wrapper only. It ran both baselines +before any mutant. The helper runs each mode's complete inventory in turn, as the +normalizer wrapper already did, and as the characterization runs measured it. +Each child is still a separate process started from the packet directory. + +## The contract + +The references are complete verdicts measured from the unchanged CLIs, never +outputs learned from the invocation under test. + +- **Baseline:** exit 0, empty stderr, and the measured `{"check": "ok", …, + "mutant": null}` verdict, which carries the certified values that `--check` has + already compared with `RESULTS.json`. +- **Each mutant:** exit 1, empty stderr, and exactly the verdict that the checker's + `require()` prints: `{"check": "FAIL", "reason": …, "mutant": M}`. + +| Family | Mutants (in run order) | Measured reason | +|---|---|---| +| `c8-cap-failure-coefficient-dims` | `no-y-tail`, `r3-one`, `no-bp`, `frob-omega-off` | `certified values differ from RESULTS.json` | +| `c8-normalizer-floor-planar` | `swap-minmax`, `drop-torus-error`, `sign-saddle` | `exact replay of low r=[63/512,1/8],b=[0,1/8],k=[1/2,3/4]` | + +Within each family every mutant fails the same first check. The verdict names +that check, and the mutant label, but not which quantity the mutant changed. That +is how these checkers report, and the packet bytes stay unchanged. The contract +therefore pins the first failing check exactly. A crash, a silent exit, stderr +output, a failure at any other check, or a verdict for another label is rejected. + +## Execution and non-vacuous regressions + +All of the following ran locally on CPython 3.11.15. The characterization and the +real-checker replay ran on #518's head `b0dd89f`. The tests and the RED control ran +again on `1965142e`. + +- **Characterization:** two separate runs of the unchanged checkers, in both + modes. Every normal and optimized stdout is byte-identical, every stderr is + empty, and the exit statuses are as above. Locally, the normalizer baseline + takes about 6 minutes per mode, each normalizer mutant 28 to 33 seconds, and + each dims child under half a second. +- **Real checkers through the helper:** `replay()` on both actual packets gave + `protocol_ok` for every child: 10 for dims in 2.5 seconds and 8 for the + normalizer in about 14.5 minutes. Every stdout digest equals the characterized + one. +- **This test file:** 13 tests OK in both modes, both as the workflow step runs it + and under the full suite's discovery, about 23 seconds per mode for both + families. +- **RED control:** the same tests, with `CHECK_MODE_SOURCE_ROOT` set to unchanged + `main`, give 90 false acceptances: + + | Test | dims | normalizer | + |---|---|---| + | crashes | 8 | 6 | + | silent exits | 8 | 6 | + | wrong reasons | 8 | 6 | + | wrong mutant label | 2 | 2 | + | baseline faults (stderr, wrong value, missing key, malformed) | 8 | 8 | + | JSON faults (stderr, duplicate, NaN, trailing, extra, missing, malformed) | 14 | 14 | + + The wrong exit statuses were already rejected there and still are. The trigger, + step and import tests fail there too, as expected, and so does the dims + inventory-order check, because the old dims wrapper ran both baselines first. + +## Limits + +- The local interpreter is 3.11.15, not the hosted 3.11.16. The hosted runs are + the qualifying execution. +- The synthetic fixture uses a one-entry pin list and a minimal manifest. The real + pins run only in the real hosted job. +- This is batch 1 of seven. `c8-cap-failure-coefficient-planar`, + `c8-lifetime-difference-constant`, `c8-theorem-a-constants-d3`, + `c8-theorem-a-constants-planar` and `c8-torus-vanishing-remainder` keep their + own stages. diff --git a/tests/test_check_mode_workflows.py b/tests/test_check_mode_workflows.py new file mode 100644 index 000000000..11ec86696 --- /dev/null +++ b/tests/test_check_mode_workflows.py @@ -0,0 +1,294 @@ +"""Synthetic whole-shell regressions for the C8 check-mode wrappers. + +The fixture commits synthetic source, manifest and upstream bytes, then runs the +actual verification step of each wrapper with real subprocesses and the measured +verdicts from tools/check_mode_contracts.json. It also checks each wrapper's +trigger paths, step order and import line. This is protocol coverage, not a +field calculation. +""" +from __future__ import annotations +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import textwrap +import unittest + +from test_legacy_workflow_triggers import covered, trigger_paths + +ROOT = Path(__file__).resolve().parents[1] +SELF = 'tests/test_check_mode_workflows.py' +SOURCE = Path(os.environ.get('CHECK_MODE_SOURCE_ROOT', str(ROOT))) +CONTRACTS = json.loads((ROOT / 'tools/check_mode_contracts.json').read_text()) +WORKFLOWS = { + 'c8-cap-failure-coefficient-dims': 'c8-cap-failure-coefficient-dims.yml', + 'c8-normalizer-floor-planar': 'c8-normalizer-floor-planar.yml', +} +ACTIVE = {k: v for k, v in CONTRACTS['families'].items() + if not os.environ.get('CHECK_MODE_TEST_FAMILY') or k == os.environ['CHECK_MODE_TEST_FAMILY']} +if not ACTIVE: + raise SystemExit('unknown test family') +# Mutant order within each mode, as in each wrapper's original loop. +ORDERS = { + 'c8-cap-failure-coefficient-dims': ('no-y-tail', 'r3-one', 'no-bp', 'frob-omega-off'), + 'c8-normalizer-floor-planar': ('swap-minmax', 'drop-torus-error', 'sign-saddle'), +} +MODES = ('normal', 'optimized') +VERIFY = ' - name: Verify manifest and main-resident pins' +STEP = ( + ' - name: Check trigger coverage and the complete shell rejection contract in both modes\n' + ' env:\n' + " CHECK_MODE_TEST_FAMILY: '%s'\n" + " PYTHONDONTWRITEBYTECODE: '1'\n" + ' run: |\n' + ' python -B -S ' + SELF + '\n' + ' python -B -O -S ' + SELF + '\n' +) + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def blob(data): + return hashlib.sha1(b'blob %d\0' % len(data) + data).hexdigest() + + +def row(path, data): + return {'path': path, 'bytes': len(data), 'sha256': digest(data), 'git_blob': blob(data)} + + +def workflow_text(family, source=SOURCE): + return (source / '.github/workflows' / WORKFLOWS[family]).read_text(encoding='utf-8') + + +def verification_shell(text): + lines = text.splitlines() + start = next(i for i, line in enumerate(lines) if line.startswith(VERIFY)) + start = next(i for i in range(start, len(lines)) if lines[i].strip() == 'run: |') + 1 + end = start + while end < len(lines) and (not lines[end].strip() or lines[end].startswith(' ')): + end += 1 + return textwrap.dedent('\n'.join(lines[start:end])) + '\n' + + +CHILD = r''' +import json, os, sys +baseline = BASELINE +reports = REPORTS +args = sys.argv[1:] +if args[:1] != ['--check'] or len(args) not in (1, 3) or (len(args) == 3 and args[1] != '--mutant'): + sys.stderr.write('unexpected arguments\n') + sys.exit(3) +label = args[2] if len(args) == 3 else 'baseline' +mode = 'optimized' if sys.flags.optimize else 'normal' +with open(os.environ['CHECK_MODE_TRACE'], 'a') as f: + f.write(json.dumps([label, mode, os.getcwd()]) + '\n') +value, rc = (baseline, 0) if label == 'baseline' else (reports[label], 1) +output = json.dumps(value) + '\n' +if os.environ.get('CHECK_MODE_TARGET') == label and os.environ.get('CHECK_MODE_MODE') == mode: + kind = os.environ.get('CHECK_MODE_FAULT', '') + if kind == 'crash': + raise RuntimeError('unrelated fixture crash') + if kind == 'silent': + sys.exit(rc) + if kind == 'stderr': + sys.stderr.write('unexpected diagnostic\n') + if kind == 'wrong-exit': + rc = 0 if label != 'baseline' else 1 + if kind == 'wrong-reason': + output = json.dumps(dict(value, reason='unrelated check failed')) + '\n' + if kind == 'wrong-label': + output = json.dumps(dict(value, mutant='another-mutant')) + '\n' + if kind == 'wrong-value': + name = next(k for k in value if k not in ('check', 'mutant')) + output = json.dumps(dict(value, **{name: 'unrelated value'})) + '\n' + if kind == 'format': + output = ' \n' + json.dumps(value, indent=4, sort_keys=True) + '\n\t' + if kind == 'duplicate': + output = output.rstrip()[:-1] + ', "check": ' + json.dumps(value['check']) + '}\n' + if kind == 'nonfinite': + output = output.rstrip()[:-1] + ', "unexpected": NaN}\n' + if kind == 'trailing': + output += '{}\n' + if kind == 'extra': + output = json.dumps(dict(value, unexpected='x')) + '\n' + if kind == 'missing': + output = json.dumps({k: v for k, v in value.items() if k != 'check'}) + '\n' + if kind == 'malformed': + output = 'not JSON\n' +sys.stdout.write(output) +sys.exit(rc) +''' + + +class CheckModeWorkflowTests(unittest.TestCase): + maxDiff = None + + def case(self, family, label='baseline', mode='normal', fault='', tamper=None): + with tempfile.TemporaryDirectory(prefix='check-mode-test-') as td: + outside = Path(td) + repo = outside / 'repo' + repo.mkdir() + trace = outside / 'trace' + contract = json.loads(json.dumps(CONTRACTS)) + entry = contract['families'][family] + directory = repo / entry['directory'] + directory.mkdir(parents=True) + script = CHILD.replace('BASELINE', repr(entry['baseline'])).replace('REPORTS', repr(entry['mutants'])) + data = script.encode() + (directory / entry['script']).write_bytes(data) + results = (ROOT / entry['directory'] / 'RESULTS.json').read_bytes() + (directory / 'RESULTS.json').write_bytes(results) + upstream = b'synthetic upstream; no scientific authority\n' + (repo / 'upstream.txt').write_bytes(upstream) + pin = dict(row('upstream.txt', upstream), current_required=True) + source_map = json.dumps({'sources': [pin]}).encode() + (directory / 'SOURCE_MAP.json').write_bytes(source_map) + files = [row(entry['script'], data), row('RESULTS.json', results), row('SOURCE_MAP.json', source_map)] + (directory / 'SOURCE_FILES.json').write_text(json.dumps({'files': files})) + (repo / 'tools').mkdir() + for helper in ('check_mode_replay.py', 'legacy_json_replay.py'): + if (SOURCE / 'tools' / helper).exists(): + shutil.copyfile(SOURCE / 'tools' / helper, repo / 'tools' / helper) + entry['script_git_blob'] = blob(data) + # Only the checker identity is re-bound for this synthetic fixture. + (repo / 'tools/check_mode_contracts.json').write_text(json.dumps(contract)) + env = os.environ.copy() + env.update({'CHECK_MODE_TRACE': str(trace), 'CHECK_MODE_TARGET': label, 'CHECK_MODE_MODE': mode, + 'CHECK_MODE_FAULT': fault, 'PYTHONDONTWRITEBYTECODE': '1'}) + for cmd in (['git', 'init', '-q'], ['git', 'add', '.'], + ['git', '-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', + '-c', 'commit.gpgsign=false', 'commit', '-qm', 'synthetic fixture']): + subprocess.run(cmd, cwd=repo, check=True, capture_output=True, timeout=15, env=env) + if tamper == 'packet': + (directory / entry['script']).write_bytes(data + b'\n') + if tamper == 'upstream': + (repo / 'upstream.txt').write_bytes(upstream + b'x') + shell = verification_shell(workflow_text(family)) + result = subprocess.run(['bash', '-c', shell], cwd=repo, env=env, capture_output=True, timeout=90) + events = [json.loads(s) for s in trace.read_text().splitlines()] if trace.exists() else [] + return result, events, str(directory.resolve()) + + def rejected(self, family, label, mode, fault): + result, events, _ = self.case(family, label, mode, fault) + self.assertNotEqual(result.returncode, 0, (family, label, mode, fault, 'false acceptance', events)) + self.assertEqual(events[-1][:2], [label, mode], + (family, label, mode, fault, events, result.stderr.decode(errors='replace'))) + + def test_reference_custody(self): + canonical = json.dumps(CONTRACTS['families'], sort_keys=True, separators=(',', ':')).encode() + self.assertEqual(digest(canonical), '9127eaa7bc8b14bf1187940fa89c0e59ad94015420a82d291147e83f3bafb2e9') + for family, entry in ACTIVE.items(): + with self.subTest(family=family): + packet = ROOT / entry['directory'] + self.assertEqual(blob((packet / entry['script']).read_bytes()), entry['script_git_blob']) + results = (packet / 'RESULTS.json').read_bytes() + self.assertEqual((len(results), digest(results)), (entry['results_bytes'], entry['results_sha256'])) + + def test_valid_complete_inventory(self): + for family in ACTIVE: + with self.subTest(family=family): + result, events, packet = self.case(family) + self.assertEqual(result.returncode, 0, result.stderr.decode(errors='replace')) + labels = ['baseline', *ORDERS[family]] + self.assertEqual(events, [[label, mode, packet] for mode in MODES for label in labels]) + + def test_formatting_alone_is_accepted(self): + for family in ACTIVE: + for label in ('baseline', ORDERS[family][0]): + with self.subTest(family=family, label=label): + result, _, _ = self.case(family, label, 'optimized', 'format') + self.assertEqual(result.returncode, 0, result.stderr.decode(errors='replace')) + + def test_crashes_each_mutant_each_mode(self): + for family in ACTIVE: + for label in ORDERS[family]: + for mode in MODES: + with self.subTest(family=family, label=label, mode=mode): + self.rejected(family, label, mode, 'crash') + + def test_silent_exits_each_mutant_each_mode(self): + for family in ACTIVE: + for label in ORDERS[family]: + for mode in MODES: + with self.subTest(family=family, label=label, mode=mode): + self.rejected(family, label, mode, 'silent') + + def test_wrong_reasons_each_mutant_each_mode(self): + for family in ACTIVE: + for label in ORDERS[family]: + for mode in MODES: + with self.subTest(family=family, label=label, mode=mode): + self.rejected(family, label, mode, 'wrong-reason') + + def test_wrong_mutant_label(self): + for family in ACTIVE: + for mode in MODES: + with self.subTest(family=family, mode=mode): + self.rejected(family, ORDERS[family][0], mode, 'wrong-label') + + def test_baseline_contract(self): + for family in ACTIVE: + for fault in ('stderr', 'wrong-value', 'wrong-exit', 'missing', 'malformed'): + for mode in MODES: + with self.subTest(family=family, fault=fault, mode=mode): + self.rejected(family, 'baseline', mode, fault) + + def test_json_contract(self): + for family in ACTIVE: + label = ORDERS[family][0] + for fault in ('stderr', 'wrong-exit', 'duplicate', 'nonfinite', 'trailing', 'extra', 'missing', + 'malformed'): + for mode in MODES: + with self.subTest(family=family, fault=fault, mode=mode): + self.rejected(family, label, mode, fault) + + def test_source_guards_run_before_any_child(self): + for family in ACTIVE: + for tamper in ('packet', 'upstream'): + with self.subTest(family=family, tamper=tamper): + result, events, _ = self.case(family, tamper=tamper) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(events, []) + + +class CheckModeTriggerTests(unittest.TestCase): + def test_paths_cover_the_helper_its_inputs_and_these_tests(self): + for family, entry in ACTIVE.items(): + paths = trigger_paths(workflow_text(family)) + for path in (entry['directory'] + '/' + entry['script'], '.github/workflows/' + WORKFLOWS[family], + 'tools/check_mode_replay.py', 'tools/check_mode_contracts.json', + 'tools/legacy_json_replay.py', 'tools/__init__.py', 'tools.py', SELF, + 'tests/test_legacy_workflow_triggers.py'): + with self.subTest(family=family, path=path): + self.assertTrue(covered(paths, path), 'consumed or regression path is uncovered') + for path in ('README.md', 'unrelated/tools.py', 'tools_extra/file.py'): + with self.subTest(family=family, unrelated=path): + self.assertFalse(covered(paths, path), 'unexpected unrelated trigger') + + def test_contract_step_runs_in_both_modes_before_the_replay(self): + for family in ACTIVE: + text = workflow_text(family) + with self.subTest(family=family): + step = STEP % family + self.assertEqual(text.count(step), 1, 'missing exact two-mode contract step') + self.assertLess(text.index(step), text.index(VERIFY)) + + def test_the_replay_is_the_only_import_and_names_its_family(self): + for family in ACTIVE: + text = workflow_text(family) + with self.subTest(family=family): + lines = [s.strip() for s in text.splitlines() if s.strip().startswith('from tools.')] + self.assertEqual(lines, ['from tools.check_mode_replay import replay']) + self.assertEqual(text.count("replay('%s', root" % family), 1) + # No exit-status-only check of a child remains beside the replay. + self.assertNotIn('.returncode', verification_shell(text)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/check_mode_contracts.json b/tools/check_mode_contracts.json new file mode 100644 index 000000000..2c25b879b --- /dev/null +++ b/tools/check_mode_contracts.json @@ -0,0 +1 @@ +{"families":{"c8-cap-failure-coefficient-dims":{"baseline":{"cG3_0":{"1":["2803036.717892","2803338.074669"],"1/2":["358497.545122","619756.489477"],"2":["22424320.655069","22424320.655072"],"3/2":["9460258.333462","9460258.334033"],"3/4":["1182683.064519","1201290.889423"]},"check":"ok","mutant":null},"directory":"frontiers/c8_cap_failure_coefficient_dims_20261001","mutants":{"frob-omega-off":{"check":"FAIL","mutant":"frob-omega-off","reason":"certified values differ from RESULTS.json"},"no-bp":{"check":"FAIL","mutant":"no-bp","reason":"certified values differ from RESULTS.json"},"no-y-tail":{"check":"FAIL","mutant":"no-y-tail","reason":"certified values differ from RESULTS.json"},"r3-one":{"check":"FAIL","mutant":"r3-one","reason":"certified values differ from RESULTS.json"}},"results_bytes":6837,"results_sha256":"1406d5138ac3d33bcb0358f6a3005a67e3a4a2dbfc2c43bc3ac2593667550608","script":"cap_coefficient_dims.py","script_git_blob":"af7e9d103f962f613c57f5e695f030165e348beb"},"c8-normalizer-floor-planar":{"baseline":{"check":"ok","mutant":null,"z_star_combined":"3.5044542729","z_star_high":"3.5044542729","z_star_low":"7.0499921959"},"directory":"frontiers/c8_normalizer_floor_planar_20260930","mutants":{"drop-torus-error":{"check":"FAIL","mutant":"drop-torus-error","reason":"exact replay of low r=[63/512,1/8],b=[0,1/8],k=[1/2,3/4]"},"sign-saddle":{"check":"FAIL","mutant":"sign-saddle","reason":"exact replay of low r=[63/512,1/8],b=[0,1/8],k=[1/2,3/4]"},"swap-minmax":{"check":"FAIL","mutant":"swap-minmax","reason":"exact replay of low r=[63/512,1/8],b=[0,1/8],k=[1/2,3/4]"}},"results_bytes":1874499,"results_sha256":"a2c4d6890cc9ab0984605f78ccf89b38d93b9515b8d70a6d5e5c3d417b4b3882","script":"floor.py","script_git_blob":"29f1b512062fda6b3565429e9fe53e91b9b3c2f8"}},"provenance":"Native unchanged check-mode verdicts of the C8 checkers, measured for Math-#322 (C8-WRAPPERS-REPAIR-C8-20261026) in both modes from each packet directory. This file is an engineering reference, not mathematical acceptance.","schema":1,"source_cut":"f0f90857dcf7586e5195fcb05c0058657fba4003"} diff --git a/tools/check_mode_replay.py b/tools/check_mode_replay.py new file mode 100644 index 000000000..59f0b8c5f --- /dev/null +++ b/tools/check_mode_replay.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Authenticate the C8 check-mode checkers' acceptance and rejection verdicts. + +Called AFTER each workflow's unchanged source and pin guards. Each checker runs +from its packet directory as `