From 9284b201bb6e8179b87fa16ac42b3a9aec6b0c89 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:40:23 -0700 Subject: [PATCH 1/2] Test-first QS replay reason controls (expected RED; no mathematical edits) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dylan Roy — delegated AI engineering. OpenAI / GPT-6 Astra Pro, d3-l4-round7-qs-replay-20261005. Scope pickup Math-307/6007516519. Eight real-subprocess test methods reproduce 27 false-acceptance subcases in each Python mode against exact original replay3dcb3fe8. Preserve RED hosted results and exact public source snapshot for the repair audit. Only the added regression and existing workflow wiring change here. All91 stored records, scripts, stdout, source manifest and wrapper unchanged. Draft engineering proposal, not a merge candidate; scientific effect NONE. --- .github/workflows/qs-d3-soft-layer-chain.yml | 29 +++- tests/test_qs_d3_replay_protocol.py | 131 +++++++++++++++++++ 2 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 tests/test_qs_d3_replay_protocol.py diff --git a/.github/workflows/qs-d3-soft-layer-chain.yml b/.github/workflows/qs-d3-soft-layer-chain.yml index d35a54ea3..2fcf36c84 100644 --- a/.github/workflows/qs-d3-soft-layer-chain.yml +++ b/.github/workflows/qs-d3-soft-layer-chain.yml @@ -4,6 +4,7 @@ name: QS d = 3 soft-layer chain (A3-A3.6) replay pull_request: paths: - 'frontiers/qs_d3_soft_layer_chain_20261005/**' + - 'tests/test_qs_d3_replay_protocol.py' - '.github/workflows/qs-d3-soft-layer-chain.yml' workflow_dispatch: @@ -26,11 +27,35 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 with: python-version: '3.11.16' - - name: Verify stored identities, extractions and replay the eight published author controls + - name: Verify protocol, stored identities, extractions and all published controls env: PYTHONDONTWRITEBYTECODE: '1' + shell: bash run: | set -euo pipefail - python ${{ matrix.mode }} frontiers/qs_d3_soft_layer_chain_20261005/replay.py + out="$RUNNER_TEMP/qs-replay-evidence" + mkdir -p "$out/logs" + python ${{ matrix.mode }} -m unittest discover -s tests -p test_qs_d3_replay_protocol.py -v 2>&1 | tee "$out/logs/protocol.txt" + python ${{ matrix.mode }} frontiers/qs_d3_soft_layer_chain_20261005/replay.py 2>&1 | tee "$out/logs/replay.txt" git diff --exit-code test -z "$(git status --porcelain)" + - name: Preserve exact public sources and available execution logs + if: always() + shell: bash + run: | + set -euo pipefail + out="$RUNNER_TEMP/qs-replay-evidence" + mkdir -p "$out" + cp -a frontiers/qs_d3_soft_layer_chain_20261005 "$out/packet" + cp tests/test_qs_d3_replay_protocol.py "$out/" + cp .github/workflows/qs-d3-soft-layer-chain.yml "$out/" + git rev-parse HEAD > "$out/tested_commit.txt" + printf '%s\n' '${{ matrix.mode }}' > "$out/mode.txt" + printf '%s\n' "$GITHUB_RUN_ID/$GITHUB_RUN_ATTEMPT" > "$out/run.txt" + - name: Retain evidence on success or failure + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: qs-replay-${{ strategy.job-index }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/qs-replay-evidence/ + if-no-files-found: error diff --git a/tests/test_qs_d3_replay_protocol.py b/tests/test_qs_d3_replay_protocol.py new file mode 100644 index 000000000..58d906a92 --- /dev/null +++ b/tests/test_qs_d3_replay_protocol.py @@ -0,0 +1,131 @@ +"""Real subprocess controls for QS replay; fixtures are not mathematical evidence.""" +import copy +import hashlib +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +WRAPPER = ROOT / 'frontiers/qs_d3_soft_layer_chain_20261005/replay.py' + + +class ReplayProtocolTests(unittest.TestCase): + def setUp(self): + spec = importlib.util.spec_from_file_location('qs_replay_subject', WRAPPER) + self.m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.m) + + @staticmethod + def baseline(): + return dict(object='A3 d>=3 hard-fibre reduction (author-side exact controls)', + counts={'Z1': 2400, 'Z2': 3600, 'Z3': 4800, 'Z4': 1200}, + mutant=None, all_pass=True, failures=[], scientific_effect='NONE') + + def invoke(self, scenario='valid'): + baseline = self.baseline() + mutant = copy.deepcopy(baseline) + mutant.update(mutant='Z2', all_pass=False, failures=["('Z2-gain', 17)"]) + invalid = {'error': 'unknown arguments'} + base_out = (json.dumps(baseline, sort_keys=True) + '\n').encode() + mut_code, inv_code = 1, 2 + base_err = mut_err = inv_err = b'' + if scenario == 'crash': + mut_code = 1; mut_err = b'Traceback: synthetic unrelated RuntimeError\n' + mut_out = b'' + else: + if scenario == 'wrong_reason': mutant['failures'] = ["('Z4', 17)"] + elif scenario == 'extra_reason': mutant['failures'].append("('setup', 3)") + elif scenario == 'wrong_mutant': mutant['mutant'] = 'Z3' + elif scenario == 'wrong_object': mutant['object'] = 'unrelated' + elif scenario == 'wrong_effect': mutant['scientific_effect'] = 'ACCEPT' + elif scenario == 'all_pass': mutant['all_pass'] = True + elif scenario == 'integer_bool': mutant['all_pass'] = 0 + elif scenario == 'empty_failures': mutant['failures'] = [] + elif scenario == 'failures_string': mutant['failures'] = "('Z2-gain', 17)" + elif scenario == 'wrong_counts': mutant['counts']['Z1'] = True + elif scenario == 'missing_counts': del mutant['counts']['Z2'] + elif scenario == 'extra_field': mutant['unexpected'] = 'x' + elif scenario == 'bad_tuple': mutant['failures'] = ['not a failure tuple'] + elif scenario == 'tuple_prefix_only': mutant['failures'] = ["('Z2-gain-forged', 17)"] + mut_out = (json.dumps(mutant, sort_keys=True) + '\n').encode() + if scenario == 'garbage': mut_out = b'unrelated error\n' + elif scenario == 'trailing': mut_out += b'garbage\n' + elif scenario == 'duplicate': mut_out = mut_out.replace(b'"all_pass": false', b'"all_pass": false, "all_pass": false') + elif scenario == 'nonfinite': mut_out = mut_out.replace(b'"Z1": 2400', b'"Z1": NaN') + elif scenario == 'overflow': mut_out = mut_out.replace(b'"Z1": 2400', b'"Z1": 1e999') + elif scenario == 'wrong_exit': mut_code = 2 + elif scenario == 'mutant_stderr': mut_err = b'synthetic unexpected stderr\n' + inv_out = (json.dumps(invalid) + '\n').encode() + if scenario == 'invalid_crash': inv_out = b''; inv_err = b'Traceback: synthetic error\n' + elif scenario == 'invalid_wrong_reason': inv_out = b'{"error":"unrelated"}\n' + elif scenario == 'invalid_extra_field': inv_out = b'{"error":"unknown arguments","extra":1}\n' + elif scenario == 'invalid_wrong_exit': inv_code = 1 + elif scenario == 'invalid_stderr': inv_err = b'unexpected stderr\n' + if scenario == 'baseline_stderr': base_err = b'unexpected stderr\n' + actual_base = base_out if scenario != 'baseline_mismatch' else b'wrong\n' + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); (root/'A3').mkdir() + script = root/'A3/author_controls.py' + # Every invocation launches this real isolated child; no subprocess mock. + script.write_text('import sys\na=sys.argv[1:]\n' + f'if not a: out,err,code={actual_base!r},{base_err!r},0\n' + f'elif a==["--mutant","Z2"]: out,err,code={mut_out!r},{mut_err!r},{mut_code}\n' + f'else: out,err,code={inv_out!r},{inv_err!r},{inv_code}\n' + 'sys.stdout.buffer.write(out)\nsys.stderr.buffer.write(err)\nsys.exit(code)\n') + (root/'A3/author_controls_stdout.json').write_bytes(base_out) + manifest = {'objects': [{'tag': 'A3', 'replays': [{ + 'name': 'a3d_exact.py', 'script': 'A3/author_controls.py', + 'stdout': 'A3/author_controls_stdout.json', 'args': [], + 'mutants': [['--mutant','Z2']], 'invalid': [['--bogus']]}]}]} + return self.m.replay_failures(root, manifest) + + def test_valid_protocol(self): + self.assertEqual(self.invoke(), []) + + def test_baseline_output_contract(self): + for scenario in ('baseline_mismatch', 'baseline_stderr'): + with self.subTest(scenario=scenario): self.assertTrue(self.invoke(scenario)) + + def test_crash_is_not_intended_rejection(self): + self.assertTrue(self.invoke('crash')) + + def test_intended_reason_and_identity(self): + for scenario in ('wrong_reason','extra_reason','wrong_mutant','wrong_object','wrong_effect', + 'empty_failures','failures_string','bad_tuple','tuple_prefix_only'): + with self.subTest(scenario=scenario): self.assertTrue(self.invoke(scenario)) + + def test_json_shape_and_types(self): + for scenario in ('all_pass','integer_bool','wrong_counts','missing_counts','extra_field', + 'garbage','trailing','duplicate','nonfinite','overflow'): + with self.subTest(scenario=scenario): self.assertTrue(self.invoke(scenario)) + + def test_mutant_exit_and_stderr(self): + for scenario in ('wrong_exit','mutant_stderr'): + with self.subTest(scenario=scenario): self.assertTrue(self.invoke(scenario)) + + def test_invalid_argument_contract(self): + for scenario in ('invalid_crash','invalid_wrong_reason','invalid_extra_field', + 'invalid_wrong_exit','invalid_stderr'): + with self.subTest(scenario=scenario): self.assertTrue(self.invoke(scenario)) + + def test_changed_constant_crash_is_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); (root/'A3_6').mkdir() + proof = b'x'*150 + (root/'A3_6/PROOF.md').write_bytes(proof) + script = ('from fractions import Fraction as F\nMUT=None\n' + 'coef = F(1, 4) if MUT == "M5" else F(5, 4)\n' + 'if coef==F(1,4): raise RuntimeError("synthetic crash")\n' + 'print("ok")\n') + (root/'A3_6/author_controls.py').write_text(script) + (root/'A3_6/author_controls_stdout.json').write_bytes(b'ok\n') + manifest = {'objects': [{'tag':'A3.6','replays':[], 'stored':[ + {'path':'A3_6/PROOF.md','bytes':len(proof),'sha256':hashlib.sha256(proof).hexdigest()}]}]} + self.assertTrue(self.m.negative_control_failures(root, manifest)) + + +if __name__ == '__main__': + unittest.main() From 03177c74532655be5702b1f238cd6c94fd1f2698 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:57:44 -0700 Subject: [PATCH 2/2] Require exact source-bound QS rejection contracts and fail before untrusted execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dylan Roy — delegated AI engineering. OpenAI / GPT-6 Astra Pro, d3-l4-round7-qs-replay-20261005; AUD-QS307-REPLAY-REASON-01. Preserve RED9284b201 and hosted failure37400381933. Final13 protocol tests pass in both modes; original wrapper fails31 expanded subcases. Capture140 genuine baseline/mutant/invalid invocations: complete stdout/stderr and exit contracts agree across modes. Pin62 known rejection contracts with readable source-bound reasons, including intentionally nonempty stderr. Require exact no-MUT changed-constant rejection; reject incomplete invocation inventories and stop before child execution on failed custody/extraction. All91 stored scientific records, scripts/stdouts, README and SOURCES unchanged. Only wrapper/tests update here; isolated draft proposal. Full combined wrapper and new hosted outcomes remain to be read back. Scientific effect NONE. --- .../qs_d3_soft_layer_chain_20261005/replay.py | 235 ++++++++++++++++-- tests/test_qs_d3_replay_protocol.py | 101 +++++++- 2 files changed, 315 insertions(+), 21 deletions(-) diff --git a/frontiers/qs_d3_soft_layer_chain_20261005/replay.py b/frontiers/qs_d3_soft_layer_chain_20261005/replay.py index 3dcb3fe82..cfc71d58a 100644 --- a/frontiers/qs_d3_soft_layer_chain_20261005/replay.py +++ b/frontiers/qs_d3_soft_layer_chain_20261005/replay.py @@ -13,8 +13,8 @@ 2. Every extracted script and stdout equals the fenced payload inside its stored controls comment, under the extraction rule. 3. Every control script runs under the current interpreter's -O/-S flags and reproduces its published stdout byte - for byte (none prints an interpreter version). Every listed mutant invocation exits 1, and every listed - invalid invocation exits 2. + for byte (none prints an interpreter version), with empty stderr. Every listed mutant or invalid invocation + must reproduce its exact source-bound rejection on BOTH captured streams, not merely exit 1 or 2. 4. Negative controls must be rejected: - a one-byte change to a stored note (A3_6/PROOF.md); - a control script with one changed constant (A3.6's hard-gap integral coefficient 5/4 replaced by 1/4). @@ -36,6 +36,185 @@ HERE = pathlib.Path(__file__).resolve().parent OWN = {'README.md', 'SOURCES.json', 'replay.py'} +# Exact rejection fingerprints from the UNCHANGED source packet at b157df1b57da41d2d75d6515b5e795b234ee7d34. +# Each key is (script path, complete argv tail); each value is (exit, stdout SHA256, +# stderr SHA256, readable reason). References are frozen independently of this run. +# All 48 mutants and 14 invalid invocations were executed and their named reasons +# checked against their source. Different checkers intentionally use different +# streams/formats. Exact fingerprints enforce those full contracts, including +# JSON structure/types, labels, counts, usage text, and empty/nonempty streams. +# No current failing output, generic exit code or broad regex creates a reference. +# Any source/runtime output drift fails closed and requires a reviewed amendment. +REJECTION_CONTRACTS = { + ('A3/author_controls.py', ('--mutant', 'Z2')): + (1, 'e04b2610f0b94857f65265e92fc2ef03f5d26c90e9bc76c87b0e60c3fe73bbe0', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Z2-gain'), + ('A3/author_controls.py', ('--mutant', 'Z3')): + (1, '15364c942daeecd42584fc0cd28568908542950e96d8905c94a8fe387080d22f', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Z3-drop'), + ('A3/author_controls.py', ('--mutant', 'Z4')): + (1, '5425d23fdf3ae54af74001f1d56f06b4cf9942d4d9258f9b26f84fcc52b2cfb3', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Z4'), + ('A3/author_controls.py', ('--bogus',)): + (2, '034489f97f641cc1afed3ce1cbcd4e992ee86bd1ecb35605d27149c59e7f37c9', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '{"error": "unknown arguments"}'), + ('A3_1/author_controls.py', ('--mutant', 'N1')): + (1, 'bdae332b611455aadc8065ec8651cb09bb49f06a60eab73839052e6fcea5d757', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'N1'), + ('A3_1/author_controls.py', ('--mutant', 'F1')): + (1, '8f637a031d7176b4859ff503e98c661c0ed11c344637d4b9f043b3d75ca9148e', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'F1-linear'), + ('A3_1/author_controls.py', ('--mutant', 'R1')): + (1, '94f45519c4c9b5ed2f891377e7b4780e6fdacddf16544251b0a426baf2538d15', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'R1'), + ('A3_1/author_controls.py', ('--mutant', 'C1')): + (1, 'ab8c4393467bd87b70446711b78122408a74cf8954c3e3dd16b3f33fefab95b9', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'C1'), + ('A3_1/author_controls.py', ('--mutant', 'H1')): + (1, '15d71c1992177a1f8334ac678726414f9fe8bf50851f42f2b547e0935415aa20', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'H1; H1-global'), + ('A3_1/author_controls.py', ('--bogus',)): + (2, '034489f97f641cc1afed3ce1cbcd4e992ee86bd1ecb35605d27149c59e7f37c9', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '{"error": "unknown arguments"}'), + ('A3_2/author_controls.py', ('--mutant', 'W1')): + (1, '62a9beb4f697c0f2d86151c91a93c8f1de3056ea2bb22c178b30c69672ddd1e3', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'W1-det'), + ('A3_2/author_controls.py', ('--mutant', 'W3')): + (1, 'fbea0daa327d6d93570ef86cea6365cd3a6ff27130c5eae4307c9d54ecc83f9b', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'W3-psi-c'), + ('A3_2/author_controls.py', ('--mutant', 'W4')): + (1, '5dd21eff08c0b14b4f413c743f7ba3ef8e44e85ae496550d3271f39df6fedff8', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'W4-hard'), + ('A3_2/author_controls.py', ('--mutant', 'W4r')): + (1, '3d62fdc97b4db41e301926538b642d03131dd2a7097a731a4c53fb02023f6da9', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'W4-extreme'), + ('A3_2/author_controls.py', ('--bogus',)): + (2, '034489f97f641cc1afed3ce1cbcd4e992ee86bd1ecb35605d27149c59e7f37c9', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '{"error": "unknown arguments"}'), + ('A3_3/author_controls.py', ('--mutant', 'M1')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '1d40f29decb2fb7edc239f430921c1826267f40b1ea902ec4569d6b2526761e7', 'K1 BD(a) violated on the explicit instance (ratio 13225/6664)'), + ('A3_3/author_controls.py', ('--mutant', 'M2')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'ab5b2910401dd46dcd01f5b8217bb506cdcf47956c01b4785ae9f02ce3d05a49', 'K2 BD(b) violated: n1=1 j=2 lhs=1 |c|=1'), + ('A3_3/author_controls.py', ('--mutant', 'M3')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '43a8d1f30540704886973e7bd777cafcbc0b8ca21e013f939eb0e026ecf2ff1d', 'K4 s^1 part is not the mixed block at entry (0,2)'), + ('A3_3/author_controls.py', ('--mutant', 'M4')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '0f33e6d22f652eb410cf4dfc6328cd166fa3de09babbf28e127bb86a587e29f6', 'K5 W3 bound violated: r=1/100 lam2=1 ratio=1380122500/60162641'), + ('A3_3/author_controls.py', ('--mutant', 'M5')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '3ae2328a110d8870008d319e30b02ed13bcd3602f027c5ba1c04bfa0f24a2645', 'K3 index additivity fails: (2,1) j=1'), + ('A3_3/author_controls.py', ('--bogus',)): + (2, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '727a88582d0e4605f5fafcfc233dc55111bde6af3c12aaf17e1a9f65f697cf4a', 'usage: a33_exact.py [--mutant M1|M2|M3|M4|M5]'), + ('A3_3/addendum_control_k5o.py', ('author_controls.py', '--mutant', 'W1')): + (1, '0f36d669163dbaf123b5b862b84258e6c03385e1b5f05cda49948a409d1c7ca3', '5ad81e75571085c09eed3eac956968afe62463ca2352b4f2e3d8580e7c20e7bb', 'O2: W_r/r^4; O2: pin Hessians'), + ('A3_3/addendum_control_k5o.py', ('author_controls.py', '--mutant', 'W2')): + (1, '0f36d669163dbaf123b5b862b84258e6c03385e1b5f05cda49948a409d1c7ca3', 'cba682aa71052ef54aec90f0f7571579b0dd273762c2cf231fcf0982b08ad5d9', 'O2: W_r/r^4; O2: not ordered with lambda2 < 0; O2: pin Hessians'), + ('A3_3/addendum_control_k5o.py', ('author_controls.py', '--mutant', 'W3')): + (2, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '26eac1668d0a95fd54a36c92aeff13007a4a145dc3a3667f6ec836bec0cc6cd1', 'usage: a33_k5o.py PATH/a33_exact.py [--mutant W1|W2]'), + ('A3_3/addendum_control_k5o.py', ()): + (2, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '26eac1668d0a95fd54a36c92aeff13007a4a145dc3a3667f6ec836bec0cc6cd1', 'usage: a33_k5o.py PATH/a33_exact.py [--mutant W1|W2]'), + ('A3_4/author_controls.py', ('--mutant', 'N1')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '8acd4b89cd7fac49fea480c701c5d79c6291bdea9558e66955ed5996c8eb8a1d', 'S1 spectral Jacobian fails: mu1=0 mu2=14/11 tau=-2/13'), + ('A3_4/author_controls.py', ('--mutant', 'N2')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '0d1941b2b1e8f1fcfd676bda8df6cf7bebb065c0b7dc94decc20fa6d516dfac6', 'S2 the monomial matrix at the lattice points is singular'), + ('A3_4/author_controls.py', ('--mutant', 'N3')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'b811a904b49d6d2ddc91698c011856f53acd1b41b97f36df308f8d8d5079cbab', 'S3 eigenframe jet formulas fail'), + ('A3_4/author_controls.py', ('--mutant', 'N4')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '66faa6cea9d23e2908f8f8bbbf4eda251cf8eb1c949d1edf5e152cdc4d3d9da4', 'S4 a_M + a_S != 12 lt'), + ('A3_4/author_controls.py', ('--mutant', 'N5')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '42645b1d864dde35dbed7465deea3ee17db948fd64e8dca32d882d89c2215258', 'S5 the strip integral is not of order delta^2: Lam=1/2 delta=1/2 ratio=397/144'), + ('A3_4/author_controls.py', ('--mutant', 'N6')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '75e7a9077ee5beed85ef3eec5a6b998f08a5596525a95dbf7a60a2e0b3ed3d4c', 'S2 the 20 contact functionals are not the distinct multi-indices of order <= 3'), + ('A3_4/author_controls.py', ('--bogus',)): + (2, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'ff172c1663e868f7cbd25928d384be3d46dd2f959d924bac2192c13f67b6aac2', 'usage: a34_exact.py [--mutant N1|N2|N3|N4|N5|N6]'), + ('A3_4/author_controls.py', ('--mutant', 'N7')): + (2, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'ff172c1663e868f7cbd25928d384be3d46dd2f959d924bac2192c13f67b6aac2', 'usage: a34_exact.py [--mutant N1|N2|N3|N4|N5|N6]'), + ('A3_4/author_controls.py', ('--mutant',)): + (2, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'ff172c1663e868f7cbd25928d384be3d46dd2f959d924bac2192c13f67b6aac2', 'usage: a34_exact.py [--mutant N1|N2|N3|N4|N5|N6]'), + ('A3_5/author_controls.py', ('--mutant', 'M1')): + (1, '2c6ab7d931ca3a040117e5359616b63defd5ff7e3e95b2336fcc881c400962dc', 'fb7bdb03d16f56f9d947d8380e3d2416c56cb23f5f84a7cd52fbca5f3d732e6a', 'F2_FW2'), + ('A3_5/author_controls.py', ('--mutant', 'M2')): + (1, '44e0e82e0d69a2d424b1ca74a13dcc3647a1b0f3a4f6be5320ebe98eddce8bb7', '6c6ecda028a985c503ae735ff1520195c2efbec40573b4922117840e81594216', 'F2_FW1'), + ('A3_5/author_controls.py', ('--mutant', 'M3')): + (1, '925de071af2eeb04f9f62ccd73ca83b13c5bc31a3b8ff573b8d66b45115f3f9d', '6067508640a48e55fe45b1686c0a3053b4df05049beac6bb341729e3c2447cc9', 'D1_prop_D3'), + ('A3_5/author_controls.py', ('--mutant', 'M4')): + (1, '1a7ca14805737057103daa5c0045ee36459ddb40837b029fce6102d187b25115', 'cc4c2e10e6d1992b1f4c6245f225282029e3e8bcc8ce17e4933cb25d48d611e6', 'T1_frobenius'), + ('A3_5/author_controls.py', ('--mutant', 'M5')): + (1, '1a7ca14805737057103daa5c0045ee36459ddb40837b029fce6102d187b25115', 'f511c4c70a9c52e6f84d2a931907dd71927eb20e5aaf9972153b08b9c03334fe', 'G1a_inner_bound'), + ('A3_5/author_controls.py', ('--mutant', 'M6')): + (1, '1a7ca14805737057103daa5c0045ee36459ddb40837b029fce6102d187b25115', 'ccf2fe89fcfad2d6886fc309b99c3ff4017ed077ce144542222daabaaca4e29d', 'G2_exponents'), + ('A3_5/author_controls.py', ('--mutant', 'M7')): + (1, '1a7ca14805737057103daa5c0045ee36459ddb40837b029fce6102d187b25115', '36bd6fdab1e967a26ade00260cd111fdc9acaeb6ab6581eb66fbb01b8cdef927', 'D1_prop_D3'), + ('A3_5/author_controls.py', ('--mutant', 'M8')): + (1, 'c54b30aef280a5ee39d3979927d909cdb5d7a9598c458bdcfc8585f09d1a15cb', 'b5937f8dde1e7ff7e5df16e757977a71f845319df30d2ec3373db3ac75e2b753', 'F2_FW1'), + ('A3_5/author_controls.py', ('--mutant', 'M9')): + (1, '132abd915b426b3381bd0d45860ccbba3e73897b83b3fca26cdcac95ab4afafa', '42a8a5c652e734ed7ef37b73fb04b98f43c9a1d07b1980b565f3c60bffff5799', 'F2_FW2'), + ('A3_5/author_controls.py', ('--mutant', 'M10')): + (1, '9f82daa09adda5d24cf9c0adde4e22a35f2aedd710d7e82cb18aa7a1b3bc070e', 'c17e81d9ac6f1140c1e4a5464621ed1f38e89b318b33eb1cc4707f5b301d1f4b', 'F2_FW4'), + ('A3_5/author_controls.py', ('--bogus',)): + (2, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '65b18625f68d8aa7b28667b23ea338d3b162126d20ea6477cc662f9246ba5cfe', 'usage: a35_exact.py [--mutant M1|...|M10]'), + ('A3_5/author_controls.py', ('--mutant', 'M11')): + (2, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '65b18625f68d8aa7b28667b23ea338d3b162126d20ea6477cc662f9246ba5cfe', 'usage: a35_exact.py [--mutant M1|...|M10]'), + ('A3_6/author_controls.py', ('--mutant', 'M1')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'ac6b3ae6d76bd82dec6636b71bfcef44906c92a43b3a94f8a2800cb783b6a888', 'FAILED: X1_chart, X12_witness, X16_endpoint_chain'), + ('A3_6/author_controls.py', ('--mutant', 'M2')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '31dae24b868f836f9a9f4ddac102788d320a57594cde796c2af692a2acf7b26c', 'FAILED: X2_jacobian'), + ('A3_6/author_controls.py', ('--mutant', 'M3')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '137f71e7a1be90f405a9becb4d5b4ec0e67db661e93c781392096831a89c1470', 'FAILED: X5_elder_floor'), + ('A3_6/author_controls.py', ('--mutant', 'M4')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '12cbede6726c438a06b28c9db6b4fbee2099c74ae6d6d7bece81770c3b6fd717', 'FAILED: X7_radius'), + ('A3_6/author_controls.py', ('--mutant', 'M5')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '754f0674ace72d8f126243f08ec5c4069efe11ea751ee585269ec9add9198dfe', 'FAILED: X9_hardgap'), + ('A3_6/author_controls.py', ('--mutant', 'M6')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'd090a99859fbe57008b1a9fdc4952fc410da58163a6151ec2ab27840ddf48301', 'FAILED: X8_levelband'), + ('A3_6/author_controls.py', ('--mutant', 'M7')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'b9d4a83f3b0f80aaa6f4cd9c1a3d6fae07dd7cfbc96f36a16ec83e97f4cc158f', 'FAILED: X11_ledgers'), + ('A3_6/author_controls.py', ('--mutant', 'M8')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '44b7327fd1d6686f1577e04be5e4df2f6b54e7b6ab7eab5fa93b4324a70cde28', 'FAILED: X13_chord'), + ('A3_6/author_controls.py', ('--mutant', 'M9')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'db57033261a452390cc359f24e523fe3b61834241b5cb0aad4132a3a4eb571c5', 'FAILED: X6_endpoint_floor'), + ('A3_6/author_controls.py', ('--mutant', 'M10')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '7d637e942a5e2ca81f824a5a0adfb7c16983d563a5b405b0030cf8d059a7e058', 'FAILED: X14_coverage'), + ('A3_6/author_controls.py', ('--mutant', 'M11')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '42b5323bf1b7a86bde8aff93beb03692b6ab8bb3ac499dd9a08b8dca6f3268ba', 'FAILED: X15_G3minus'), + ('A3_6/author_controls.py', ('--mutant', 'M12')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '8865b9662578908c6efbd521ebc7cf81a95f437ae4c3787908f9ead73a15345c', 'FAILED: X17_inclusions'), + ('A3_6/author_controls.py', ('--mutant', 'M13')): + (1, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '678e7e25a484af0176e0cdf3481bc96d72710fcb92680662e4740f79a6952dca', 'FAILED: X13_chord'), + ('A3_6/author_controls.py', ('--bogus',)): + (2, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '230b5ecba1d2582ee0d266ad94e94441c000ff845246535c4c88289d8c454610', 'usage: a36_exact.py [--mutant M1..M13]'), + ('A3_6/author_controls.py', ('--mutant', 'M14')): + (2, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '230b5ecba1d2582ee0d266ad94e94441c000ff845246535c4c88289d8c454610', 'usage: a36_exact.py [--mutant M1..M13]'), + ('A3_6/author_controls.py', ('--mutant',)): + (2, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', '230b5ecba1d2582ee0d266ad94e94441c000ff845246535c4c88289d8c454610', 'usage: a36_exact.py [--mutant M1..M13]'), +} + +# Replacing A3.6's 5/4 by 1/4 with MUT=None omits the usual mutant-description line. +# This separately executed altered-source control must fail specifically at X9. +NEGATIVE_HARDGAP = (1, hashlib.sha256(b'').hexdigest(), + hashlib.sha256(b'FAILED: X9_hardgap\n').hexdigest(), 'X9_hardgap') + + +def check_rejection(result, expected): + """Compare both full captured streams, not a permissive failure-message pattern.""" + code, out, err = result + failures = [] + if type(code) is not int or code != expected[0]: + failures.append('wrong exit code') + if type(out) is not bytes or hashlib.sha256(out).hexdigest() != expected[1]: + failures.append('stdout differs from intended diagnostic') + if type(err) is not bytes or hashlib.sha256(err).hexdigest() != expected[2]: + failures.append('stderr differs from intended diagnostic') + return failures + + +def rejection_inventory_failures(manifest): + listed = {} + for obj in manifest['objects']: + for replay in obj['replays']: + for kind, code in (('mutants', 1), ('invalid', 2)): + for args in replay[kind]: + key = replay['script'], tuple(args) + if key in listed: + return ['duplicate rejection invocation: ' + repr(key)] + listed[key] = code + if set(listed) != set(REJECTION_CONTRACTS): + return ['rejection invocation inventory differs from frozen contracts'] + if any(REJECTION_CONTRACTS[key][0] != code for key, code in listed.items()): + return ['rejection invocation category differs from frozen contracts'] + return [] + + +def report_control(script, args, result, reason): + code, out, err = result + print(json.dumps(dict(script=script, args=list(args), returncode=code, + stdout_sha256=hashlib.sha256(out).hexdigest(), + stderr_sha256=hashlib.sha256(err).hexdigest(), + expected_reason=reason), sort_keys=True)) + def identity(data): return len(data), hashlib.sha256(data).hexdigest() @@ -103,28 +282,37 @@ def extraction_failures(root, manifest): def run(script, *args): proc = subprocess.run([sys.executable, *interpreter_flags(), str(script), *args], capture_output=True, timeout=1800, cwd=str(script.parent)) - return proc.returncode, proc.stdout + return proc.returncode, proc.stdout, proc.stderr def replay_failures(root, manifest): - failures = [] + failures = rejection_inventory_failures(manifest) + if failures: + return failures # Do not execute an incomplete/changed control inventory. for obj in manifest['objects']: for replay in obj['replays']: label = '%s %s' % (obj['tag'], replay['name']) script = root / replay['script'] - code, out = run(script, *replay['args']) + code, out, err = run(script, *replay['args']) + baseline_failures = [] if code != 0: - failures.append('%s: exit code %s' % (label, code)) + baseline_failures.append('%s: exit code %s' % (label, code)) if out != (root / replay['stdout']).read_bytes(): - failures.append('%s: stdout differs from the published bytes' % label) - for mutant in replay['mutants']: - code, _ = run(script, *mutant) - if code != 1: - failures.append('%s: mutant %s exited %s, expected 1' % (label, ' '.join(mutant), code)) - for invalid in replay['invalid']: - code, _ = run(script, *invalid) - if code != 2: - failures.append('%s: invalid invocation %r exited %s, expected 2' % (label, ' '.join(invalid), code)) + baseline_failures.append('%s: stdout differs from the published bytes' % label) + if err: + baseline_failures.append('%s: unexpected baseline stderr' % label) + if baseline_failures: + failures.extend(baseline_failures) + continue + report_control(replay['script'], replay['args'], (code, out, err), 'exact published baseline') + for kind in ('mutants', 'invalid'): + for args in replay[kind]: + expected = REJECTION_CONTRACTS[replay['script'], tuple(args)] + result = run(script, *args) + mismatches = check_rejection(result, expected) + failures.extend('%s %r: %s' % (label, args, why) for why in mismatches) + if not mismatches: + report_control(replay['script'], args, result, expected[3]) return failures @@ -146,16 +334,23 @@ def negative_control_failures(root, manifest): failures.append('negative control: the hard-gap coefficient line was not found exactly once') else: script.write_text(text.replace(old, 'coef = F(1, 4) if MUT == "M5" else F(1, 4)'), encoding='utf-8') - code, out = run(script) - if code == 0 or out == (root / 'A3_6' / 'author_controls_stdout.json').read_bytes(): - failures.append('negative control: a checker with 5/4 replaced by 1/4 was not rejected') + result = run(script) + mismatches = check_rejection(result, NEGATIVE_HARDGAP) + failures.extend('negative control: changed hard-gap constant: ' + why for why in mismatches) + if not mismatches: + report_control('A3_6/author_controls.py', [], result, 'changed 5/4 to 1/4: X9_hardgap') return failures def main(): manifest = json.loads((HERE / 'SOURCES.json').read_text(encoding='utf-8')) - failures = (tree_failures(HERE, manifest) + extraction_failures(HERE, manifest) + replay_failures(HERE, manifest) - + negative_control_failures(HERE, manifest)) + failures = tree_failures(HERE, manifest) + if not failures: + failures = extraction_failures(HERE, manifest) + if not failures: + failures = replay_failures(HERE, manifest) + if not failures: + failures = negative_control_failures(HERE, manifest) n_scripts = sum(len(o['replays']) for o in manifest['objects']) n_mut = sum(len(r['mutants']) for o in manifest['objects'] for r in o['replays']) n_inv = sum(len(r['invalid']) for o in manifest['objects'] for r in o['replays']) diff --git a/tests/test_qs_d3_replay_protocol.py b/tests/test_qs_d3_replay_protocol.py index 58d906a92..59c24c9e8 100644 --- a/tests/test_qs_d3_replay_protocol.py +++ b/tests/test_qs_d3_replay_protocol.py @@ -7,6 +7,7 @@ import sys import tempfile import unittest +from unittest import mock ROOT = Path(__file__).resolve().parents[1] WRAPPER = ROOT / 'frontiers/qs_d3_soft_layer_chain_20261005/replay.py' @@ -29,6 +30,17 @@ def invoke(self, scenario='valid'): mutant = copy.deepcopy(baseline) mutant.update(mutant='Z2', all_pass=False, failures=["('Z2-gain', 17)"]) invalid = {'error': 'unknown arguments'} + sha = lambda data: hashlib.sha256(data).hexdigest() + # These synthetic expectations test the real runner/matcher, not the real + # checkers' reference values. The immutable production table is checked below. + expected_mutant = (json.dumps(mutant, sort_keys=True) + '\n').encode() + expected_invalid = (json.dumps(invalid) + '\n').encode() + contracts = { + ('A3/author_controls.py', ('--mutant', 'Z2')): + (1, sha(expected_mutant), sha(b''), 'Z2-gain'), + ('A3/author_controls.py', ('--bogus',)): + (2, sha(expected_invalid), sha(b''), 'unknown arguments'), + } base_out = (json.dumps(baseline, sort_keys=True) + '\n').encode() mut_code, inv_code = 1, 2 base_err = mut_err = inv_err = b'' @@ -80,7 +92,8 @@ def invoke(self, scenario='valid'): 'name': 'a3d_exact.py', 'script': 'A3/author_controls.py', 'stdout': 'A3/author_controls_stdout.json', 'args': [], 'mutants': [['--mutant','Z2']], 'invalid': [['--bogus']]}]}]} - return self.m.replay_failures(root, manifest) + with mock.patch.object(self.m, 'REJECTION_CONTRACTS', contracts, create=True): + return self.m.replay_failures(root, manifest) def test_valid_protocol(self): self.assertEqual(self.invoke(), []) @@ -127,5 +140,91 @@ def test_changed_constant_crash_is_rejected(self): self.assertTrue(self.m.negative_control_failures(root, manifest)) +class ReferenceContractTests(unittest.TestCase): + def setUp(self): + spec = importlib.util.spec_from_file_location('qs_reference_subject', WRAPPER) + self.m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.m) + + def matcher(self): + self.assertTrue(hasattr(self.m, 'check_rejection'), 'missing exact diagnostic matcher') + return self.m.check_rejection + + def test_fixed_table_covers_every_manifest_invocation(self): + self.assertTrue(hasattr(self.m, 'REJECTION_CONTRACTS'), 'missing frozen reference table') + manifest = json.loads(WRAPPER.with_name('SOURCES.json').read_text()) + listed = {} + for obj in manifest['objects']: + for replay in obj['replays']: + for category, code in (('mutants', 1), ('invalid', 2)): + for args in replay[category]: + key = replay['script'], tuple(args) + self.assertNotIn(key, listed) + listed[key] = code + self.assertEqual(len(listed), 62) + self.assertEqual(set(self.m.REJECTION_CONTRACTS), set(listed)) + for key, value in self.m.REJECTION_CONTRACTS.items(): + self.assertEqual(len(value), 4) + self.assertEqual(value[0], listed[key]) + for digest in value[1:3]: self.assertRegex(digest, r'^[0-9a-f]{64}$') + self.assertIsInstance(value[3], str) + self.assertTrue(value[3]) + + def test_exact_matcher_preserves_five_diagnostic_families(self): + check = self.matcher() + # Representative valid shapes. Full exact values come from unchanged sources, + # not a permissive JSON/regex schema or the current child's reported result. + shapes = [ + (1, b'{"all_pass":false,"mutant":"Z2","failures":["Z2-gain"]}\n', b''), + (1, b'', b'{"failed":"K1 BD(a) violated"}\n'), + (1, b'{"passed":false}\n', b'FAILED:\nO2: pin Hessians\n'), + (1, b'{"passed":false}\n', b'FAILED:\nF2_FW1: ratio 2\n'), + (1, b'', b'FAILED: X9_hardgap\nmutant M5: hard-gap integral\n'), + (2, b'{"error": "unknown arguments"}\n', b''), + (2, b'', b'usage: a33_k5o.py PATH/a33_exact.py [--mutant W1|W2]\n'), + ] + for code, out, err in shapes: + expected = (code, hashlib.sha256(out).hexdigest(), hashlib.sha256(err).hexdigest(), 'synthetic intended reason') + with self.subTest(code=code, out=out, err=err): + self.assertEqual(check((code, out, err), expected), []) + for bad in ((0, out, err), (code, out+b'x', err), (code, out, err+b'x'), + (code, err, out), (code, b'', b'Traceback: unrelated error')): + if bad == (code,out,err): continue + self.assertTrue(check(bad, expected)) + + def test_table_cannot_omit_or_duplicate_manifest_controls(self): + self.assertTrue(hasattr(self.m, 'REJECTION_CONTRACTS'), 'missing inventory guard') + manifest = json.loads(WRAPPER.with_name('SOURCES.json').read_text()) + obj = manifest['objects'][0] + for variant in ('missing', 'duplicate', 'unlisted'): + bad = copy.deepcopy(manifest) + args = bad['objects'][0]['replays'][0]['mutants'] + if variant == 'missing': args.pop() + elif variant == 'duplicate': args.append(args[0]) + else: args.append(['--mutant', 'unlisted']) + with self.subTest(variant=variant), mock.patch.object(self.m, 'run', side_effect=AssertionError('child before inventory')): + self.assertTrue(self.m.replay_failures(WRAPPER.parent, bad)) + + def test_invalid_custody_never_executes_a_child(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root/'SOURCES.json').write_text('{"objects":[]}') + with mock.patch.object(self.m, 'HERE', root), \ + mock.patch.object(self.m, 'replay_failures', side_effect=AssertionError('execution before custody')), \ + mock.patch.object(self.m, 'negative_control_failures', side_effect=AssertionError('control before custody')): + self.assertEqual(self.m.main(), 1) + + def test_workflow_runs_regressions_and_retains_failed_evidence(self): + text = (ROOT/'.github/workflows/qs-d3-soft-layer-chain.yml').read_text() + self.assertIn("'tests/test_qs_d3_replay_protocol.py'", text) + self.assertIn('-m unittest discover -s tests -p test_qs_d3_replay_protocol.py -v', text) + self.assertIn('set -euo pipefail', text) + self.assertIn('if: always()', text) + self.assertIn('if-no-files-found: error', text) + self.assertIn('git diff --exit-code', text) + self.assertIn('git status --porcelain', text) + self.assertNotIn('continue-on-error', text) + + if __name__ == '__main__': unittest.main()