From cdaaf62433430757e3899948b2d59bde6471df76 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:45:34 -0700 Subject: [PATCH] Preserve downstream validation timeout evidence without weakening gates --- .github/workflows/downstream-gate.yml | 4 + .../SOURCE_FILES.json | 4 +- .../run_validation.py | 45 +++- tests/test_downstream_validation.py | 219 ++++++++++++++++++ 4 files changed, 265 insertions(+), 7 deletions(-) create mode 100644 tests/test_downstream_validation.py diff --git a/.github/workflows/downstream-gate.yml b/.github/workflows/downstream-gate.yml index 8e86eee86..13ba3b5c2 100644 --- a/.github/workflows/downstream-gate.yml +++ b/.github/workflows/downstream-gate.yml @@ -32,6 +32,10 @@ jobs: run: | python3 -B -S -m unittest discover -s tests -p test_required_formal_check.py -v python3 -B -O -S -m unittest discover -s tests -p test_required_formal_check.py -v + - name: Validation timeout evidence controls in both Python modes + run: | + python3 -B -S -m unittest discover -s tests -p test_downstream_validation.py -v + python3 -B -O -S -m unittest discover -s tests -p test_downstream_validation.py -v - name: Verify exact sources, transition inputs and regressions env: PYTHONDONTWRITEBYTECODE: '1' diff --git a/frontiers/downstream_gate_20260925/SOURCE_FILES.json b/frontiers/downstream_gate_20260925/SOURCE_FILES.json index 1fa1e59e4..c6bbba669 100644 --- a/frontiers/downstream_gate_20260925/SOURCE_FILES.json +++ b/frontiers/downstream_gate_20260925/SOURCE_FILES.json @@ -37,8 +37,8 @@ "sha256": "84b7ad724e4da1c5b4b396c90ae03d4ddedc37771a6e45e1082a45fd80d7ff4a" }, "run_validation.py": { - "bytes": 7517, - "sha256": "276e69c2b29e012c386db615ed64e24badf3726a9f122fd9d77776a9c4f00711" + "bytes": 9465, + "sha256": "55c111cf4088b2511d9ab20ce8a8cc36a008c91dbbf140377539a5452b531215" }, "test_hard_gate.py": { "bytes": 23161, diff --git a/frontiers/downstream_gate_20260925/run_validation.py b/frontiers/downstream_gate_20260925/run_validation.py index 58c2b5e85..e4e771140 100644 --- a/frontiers/downstream_gate_20260925/run_validation.py +++ b/frontiers/downstream_gate_20260925/run_validation.py @@ -6,6 +6,7 @@ import shutil import subprocess import sys +import time ROOT = Path(__file__).resolve().parent EXPECTED_TESTS = 71 @@ -63,8 +64,36 @@ def identities(): for p in sorted(ROOT.rglob('*')) if p.is_file() and '__pycache__' not in p.parts} -def execute(command, cwd, out, name, mutant=False): - result = subprocess.run(command, cwd=cwd, capture_output=True, text=True, timeout=30) +def execute(command, cwd, out, name, mutant=False, *, mode, mutation=None): + timeout_seconds = 30 + started = time.monotonic() + try: + result = subprocess.run(command, cwd=cwd, capture_output=True, text=True, timeout=timeout_seconds) + except subprocess.TimeoutExpired as timeout: + elapsed = time.monotonic() - started + failures = [] + streams = {} + # TimeoutExpired streams are bytes even with text=True. In particular, + # a killed child can leave an incomplete UTF-8 sequence: never decode it. + for stream, raw in [('stdout', timeout.stdout), ('stderr', timeout.stderr)]: + streams[stream] = {'present': raw is not None, 'bytes': len(raw) if raw is not None else 0} + path = out/(name+'.'+stream) + try: + path.write_bytes(raw if raw is not None else b'') + except Exception as error: + failure = f'{path.name}: {type(error).__name__}: {error}' + failures.append(failure) + timeout.add_note('Timeout evidence persistence failed: '+failure) + record = {'argv': list(command), 'cwd': str(Path(cwd).resolve()), + 'run_name': name, 'mode': mode, 'mutation': mutation, 'mutant': mutant, + 'timeout_seconds': timeout_seconds, 'elapsed_seconds': elapsed, + 'status': 'timed_out', 'streams': streams, 'persistence_failures': failures} + path = out/(name+'.timeout.json') + try: + path.write_text(json.dumps(record, sort_keys=True, indent=2)+'\n') + except Exception as error: + timeout.add_note(f'Timeout evidence persistence failed: {path.name}: {type(error).__name__}: {error}') + raise (out/(name+'.stdout')).write_text(result.stdout) (out/(name+'.stderr')).write_text(result.stderr) require(f'Ran {EXPECTED_TESTS} tests' in result.stderr and 'skipped=' not in result.stderr, 'unexpected test coverage: '+name) @@ -92,7 +121,7 @@ def main(): try: for mode, flags in [('normal', []), ('optimized', ['-O'])]: cmd = [sys.executable, '-B', *flags, '-S', '-m', 'unittest', 'discover', '-p', 'test_*.py', '-v'] - execute(cmd, ROOT, out, 'tests_'+mode) + execute(cmd, ROOT, out, 'tests_'+mode, mode=mode) result = subprocess.run([sys.executable, '-B', *flags, '-S', 'hard_gate.py'], cwd=ROOT, capture_output=True, timeout=15) require(result.returncode == 0 and result.stdout == (ROOT/'RESULTS.json').read_bytes(), 'entry/result mismatch: '+mode) (out/('output_'+mode+'.json')).write_bytes(result.stdout) @@ -103,12 +132,18 @@ def main(): scratch = out/'mutants'/mode/name shutil.copytree(ROOT, scratch, ignore=shutil.ignore_patterns('__pycache__')) (scratch/'hard_gate.py').write_text(mutated) - execute(cmd, scratch, out, 'mutation_'+mode+'_'+name, mutant=True) + execute(cmd, scratch, out, 'mutation_'+mode+'_'+name, mutant=True, mode=mode, mutation=name) report['modes'].append(mode) require(before == identities(), 'source files changed during replay') report.update(passed=True, sources_unchanged=True, source_files=before) finally: - (out/'REPORT.json').write_text(json.dumps(report, sort_keys=True, indent=2)+'\n') + active_error = sys.exc_info()[1] + try: + (out/'REPORT.json').write_text(json.dumps(report, sort_keys=True, indent=2)+'\n') + except Exception as error: + if not isinstance(active_error, subprocess.TimeoutExpired): + raise + active_error.add_note(f'Timeout evidence persistence failed: REPORT.json: {type(error).__name__}: {error}') print(json.dumps(report, sort_keys=True)) diff --git a/tests/test_downstream_validation.py b/tests/test_downstream_validation.py new file mode 100644 index 000000000..1a82a5597 --- /dev/null +++ b/tests/test_downstream_validation.py @@ -0,0 +1,219 @@ +"""Timeout diagnostics must retain evidence without weakening the replay gate.""" +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / 'frontiers/downstream_gate_20260925/run_validation.py' +SPEC = importlib.util.spec_from_file_location('downstream_validation', SCRIPT) +M = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(M) + + +class ValidationTimeoutTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.out = Path(self.temp.name) + self.command = [sys.executable, '-B', '-S', 'test.py'] + + def test_partial_binary_streams_survive_timeout_unchanged(self): + # Without the timeout handler, neither partial stream reaches disk. + stdout, stderr = b'partial stdout\x00\xe2\x82', b'partial stderr\xff\n' + timeout = subprocess.TimeoutExpired(self.command, 30, output=stdout, stderr=stderr) + with mock.patch.object(M.subprocess, 'run', side_effect=timeout): + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.execute(self.command, self.out, self.out, 'tests_normal', mode='normal') + self.assertIs(caught.exception, timeout) + self.assertTrue((self.out / 'tests_normal.stdout').is_file(), 'partial stdout was discarded') + self.assertEqual((self.out / 'tests_normal.stdout').read_bytes(), stdout) + self.assertEqual((self.out / 'tests_normal.stderr').read_bytes(), stderr) + + def test_timeout_metadata_has_exact_invocation_and_elapsed_time(self): + command = [sys.executable, '-B', '-O', '-S', '-m', 'unittest'] + timeout = subprocess.TimeoutExpired(command, 30, output=b'out', stderr=b'err') + with mock.patch.object(M.subprocess, 'run', side_effect=timeout) as run: + with mock.patch.object(M.time, 'monotonic', side_effect=[100.0, 130.25]): + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.execute(command, self.out, self.out, 'mutation_optimized_allow_green_ci', + mutant=True, mode='optimized', mutation='allow_green_ci') + self.assertIs(caught.exception, timeout) + self.assertEqual(run.call_args.kwargs['timeout'], 30) + self.assertTrue(run.call_args.kwargs['capture_output']) + self.assertTrue(run.call_args.kwargs['text']) + record = json.loads((self.out / 'mutation_optimized_allow_green_ci.timeout.json').read_text()) + self.assertEqual(record, { + 'argv': command, 'cwd': str(self.out), + 'run_name': 'mutation_optimized_allow_green_ci', 'mode': 'optimized', + 'mutation': 'allow_green_ci', 'mutant': True, + 'timeout_seconds': 30, 'elapsed_seconds': 30.25, 'status': 'timed_out', + 'streams': {'stdout': {'present': True, 'bytes': 3}, + 'stderr': {'present': True, 'bytes': 3}}, + 'persistence_failures': [], + }) + + def test_none_and_empty_streams_are_distinguished(self): + for label, payload, present in [('none', None, False), ('empty', b'', True)]: + with self.subTest(label=label): + timeout = subprocess.TimeoutExpired(self.command, 30, output=payload, stderr=payload) + with mock.patch.object(M.subprocess, 'run', side_effect=timeout): + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.execute(self.command, self.out, self.out, label, mode='normal') + self.assertIs(caught.exception, timeout) + self.assertEqual((self.out / (label + '.stdout')).read_bytes(), b'') + self.assertEqual((self.out / (label + '.stderr')).read_bytes(), b'') + record = json.loads((self.out / (label + '.timeout.json')).read_text()) + for stream in ('stdout', 'stderr'): + self.assertEqual(record['streams'][stream], {'present': present, 'bytes': 0}) + self.assertIsNone(record['mutation']) + self.assertFalse(record['mutant']) + + def test_failed_stream_write_preserves_other_stream_and_original_timeout(self): + (self.out / 'broken.stdout').mkdir() # A deterministic real filesystem failure. + timeout = subprocess.TimeoutExpired(self.command, 30, output=b'out', stderr=b'err') + with mock.patch.object(M.subprocess, 'run', side_effect=timeout): + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.execute(self.command, self.out, self.out, 'broken', mode='normal') + self.assertIs(caught.exception, timeout) + self.assertEqual((self.out / 'broken.stderr').read_bytes(), b'err') + record = json.loads((self.out / 'broken.timeout.json').read_text()) + self.assertEqual(len(record['persistence_failures']), 1) + self.assertIn('broken.stdout', record['persistence_failures'][0]) + self.assertIn('IsADirectoryError', record['persistence_failures'][0]) + self.assertTrue(any('broken.stdout' in note for note in timeout.__notes__)) + + def test_failed_metadata_write_does_not_mask_original_timeout(self): + (self.out / 'broken.timeout.json').mkdir() + timeout = subprocess.TimeoutExpired(self.command, 30, output=b'out', stderr=b'err') + with mock.patch.object(M.subprocess, 'run', side_effect=timeout): + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.execute(self.command, self.out, self.out, 'broken', mode='normal') + self.assertIs(caught.exception, timeout) + self.assertEqual((self.out / 'broken.stdout').read_bytes(), b'out') + self.assertEqual((self.out / 'broken.stderr').read_bytes(), b'err') + self.assertTrue(any('broken.timeout.json' in note for note in timeout.__notes__)) + + def test_other_subprocess_errors_are_not_caught_as_timeouts(self): + error = FileNotFoundError('missing executable') + with mock.patch.object(M.subprocess, 'run', side_effect=error): + with self.assertRaises(FileNotFoundError) as caught: + M.execute(self.command, self.out, self.out, 'missing', mode='normal') + self.assertIs(caught.exception, error) + self.assertEqual(list(self.out.iterdir()), []) + + def test_completed_baseline_behavior_and_outputs_are_unchanged(self): + completed = subprocess.CompletedProcess(self.command, 0, 'baseline stdout', 'Ran 71 tests\nOK\n') + with mock.patch.object(M.subprocess, 'run', return_value=completed): + M.execute(self.command, self.out, self.out, 'baseline', mode='normal') + self.assertEqual((self.out / 'baseline.stdout').read_text(), completed.stdout) + self.assertEqual((self.out / 'baseline.stderr').read_text(), completed.stderr) + self.assertFalse((self.out / 'baseline.timeout.json').exists()) + + def test_completed_assertion_rejection_is_still_expected_only_for_mutants(self): + stderr = 'Ran 71 tests\nAssertionError: changed semantics\nFAILED (failures=1)\n' + completed = subprocess.CompletedProcess(self.command, 1, '', stderr) + with mock.patch.object(M.subprocess, 'run', return_value=completed): + M.execute(self.command, self.out, self.out, 'mutant', mutant=True, + mode='normal', mutation='allow_green_ci') + with self.assertRaisesRegex(RuntimeError, 'baseline failed'): + M.execute(self.command, self.out, self.out, 'baseline', mode='normal') + self.assertFalse((self.out / 'mutant.timeout.json').exists()) + + def test_completed_bad_coverage_or_program_errors_still_fail(self): + bad_stderr = [ + 'Ran 70 tests\nAssertionError\nFAILED (failures=1)\n', + 'Ran 71 tests\nAssertionError\nFAILED (failures=1, skipped=1)\n', + 'Ran 71 tests\nSyntaxError\nAssertionError\nFAILED (failures=1)\n', + 'Ran 71 tests\nImportError\nAssertionError\nFAILED (failures=1)\n', + 'Ran 71 tests\nERROR: module\nAssertionError\nFAILED (failures=1)\n', + 'Ran 71 tests\nFAILED (errors=1)\n', + 'Ran 71 tests\nOK\n', + ] + for stderr in bad_stderr: + with self.subTest(stderr=stderr): + completed = subprocess.CompletedProcess(self.command, 1, '', stderr) + with mock.patch.object(M.subprocess, 'run', return_value=completed): + with self.assertRaises(RuntimeError): + M.execute(self.command, self.out, self.out, 'bad', mutant=True, + mode='normal', mutation='allow_green_ci') + + def test_main_timeout_keeps_failed_report_and_stops_later_commands(self): + timeout = subprocess.TimeoutExpired(self.command, 30, output=b'partial', stderr=b'progress') + for mutant_timeout in (False, True): + with self.subTest(mutant_timeout=mutant_timeout): + out = self.out / str(mutant_timeout) + argv = [str(SCRIPT), '--output', str(out)] + baseline = subprocess.CompletedProcess(self.command, 0, '', 'Ran 71 tests\nOK\n') + gate = subprocess.CompletedProcess(self.command, 0, (M.ROOT / 'RESULTS.json').read_bytes(), b'') + responses = [baseline, gate, timeout] if mutant_timeout else [timeout] + with mock.patch.object(sys, 'argv', argv): + with mock.patch.object(M.subprocess, 'run', side_effect=responses) as run: + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.main() + self.assertIs(caught.exception, timeout) + self.assertEqual(run.call_count, len(responses)) + report = json.loads((out / 'REPORT.json').read_text()) + self.assertFalse(report['passed']) + self.assertEqual(report['modes'], []) + self.assertFalse(report['promotion_permission']) + self.assertEqual(report['distinct_tests'], 71) + self.assertEqual(report['distinct_semantic_mutations'], 24) + self.assertEqual(report['scientific_effect'], 'NONE') + name = 'mutation_normal_bypass_own_node_eligibility' if mutant_timeout else 'tests_normal' + self.assertEqual((out / (name + '.stdout')).read_bytes(), b'partial') + record = json.loads((out / (name + '.timeout.json')).read_text()) + self.assertEqual(record['mutant'], mutant_timeout) + self.assertEqual(record['mode'], 'normal') + self.assertEqual(record['mutation'], 'bypass_own_node_eligibility' if mutant_timeout else None) + self.assertFalse((out / 'tests_optimized.stdout').exists()) + + def test_failed_final_report_write_does_not_mask_timeout(self): + timeout = subprocess.TimeoutExpired(self.command, 30, output=b'out', stderr=b'err') + out = self.out / 'main' + real_write = Path.write_text + def fail_report(path, data, *args, **kwargs): + if path.name == 'REPORT.json': + raise OSError('report disk unavailable') + return real_write(path, data, *args, **kwargs) + with mock.patch.object(sys, 'argv', [str(SCRIPT), '--output', str(out)]): + with mock.patch.object(M.subprocess, 'run', side_effect=timeout): + with mock.patch.object(Path, 'write_text', fail_report): + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.main() + self.assertIs(caught.exception, timeout) + self.assertTrue(any('REPORT.json' in note for note in timeout.__notes__)) + self.assertTrue((out / 'tests_normal.timeout.json').is_file()) + self.assertFalse((out / 'REPORT.json').exists()) + + def test_real_child_partial_bytes_are_retained(self): + # Only the test wrapper shortens the timeout. Production still supplies 30. + real_run = subprocess.run + observed = [] + def short_run(command, **kwargs): + self.assertEqual(kwargs['timeout'], 30) + kwargs['timeout'] = 0.5 + try: + return real_run(command, **kwargs) + except subprocess.TimeoutExpired as exc: + observed.append(exc) + raise + command = [sys.executable, '-B', '-S', '-c', + "import os,time; os.write(1,b'out\\xe2\\x82'); os.write(2,b'err\\xff'); time.sleep(10)"] + with mock.patch.object(M.subprocess, 'run', side_effect=short_run): + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.execute(command, self.out, self.out, 'real', mode='normal') + self.assertIs(caught.exception, observed[0]) + self.assertEqual((self.out / 'real.stdout').read_bytes(), b'out\xe2\x82') + self.assertEqual((self.out / 'real.stderr').read_bytes(), b'err\xff') + record = json.loads((self.out / 'real.timeout.json').read_text()) + self.assertEqual(record['timeout_seconds'], 30) + self.assertGreater(record['elapsed_seconds'], 0) + + +if __name__ == '__main__': + unittest.main()