diff --git a/.github/workflows/downstream-gate.yml b/.github/workflows/downstream-gate.yml index 8e86eee86..13ba3b5c2 100644 --- a/.github/workflows/downstream-gate.yml +++ b/.github/workflows/downstream-gate.yml @@ -32,6 +32,10 @@ jobs: run: | python3 -B -S -m unittest discover -s tests -p test_required_formal_check.py -v python3 -B -O -S -m unittest discover -s tests -p test_required_formal_check.py -v + - name: Validation timeout evidence controls in both Python modes + run: | + python3 -B -S -m unittest discover -s tests -p test_downstream_validation.py -v + python3 -B -O -S -m unittest discover -s tests -p test_downstream_validation.py -v - name: Verify exact sources, transition inputs and regressions env: PYTHONDONTWRITEBYTECODE: '1' diff --git a/frontiers/downstream_gate_20260925/SOURCE_FILES.json b/frontiers/downstream_gate_20260925/SOURCE_FILES.json index 1fa1e59e4..c42e11304 100644 --- a/frontiers/downstream_gate_20260925/SOURCE_FILES.json +++ b/frontiers/downstream_gate_20260925/SOURCE_FILES.json @@ -37,8 +37,8 @@ "sha256": "84b7ad724e4da1c5b4b396c90ae03d4ddedc37771a6e45e1082a45fd80d7ff4a" }, "run_validation.py": { - "bytes": 7517, - "sha256": "276e69c2b29e012c386db615ed64e24badf3726a9f122fd9d77776a9c4f00711" + "bytes": 10454, + "sha256": "ec1d3315b0c21f4608ab7b824f45d7d6e1041555f62e77feeec2bacaa8e5a9f5" }, "test_hard_gate.py": { "bytes": 23161, diff --git a/frontiers/downstream_gate_20260925/run_validation.py b/frontiers/downstream_gate_20260925/run_validation.py index 58c2b5e85..fcf648052 100644 --- a/frontiers/downstream_gate_20260925/run_validation.py +++ b/frontiers/downstream_gate_20260925/run_validation.py @@ -3,9 +3,11 @@ import argparse import hashlib import json +import math import shutil import subprocess import sys +import time ROOT = Path(__file__).resolve().parent EXPECTED_TESTS = 71 @@ -63,8 +65,55 @@ def identities(): for p in sorted(ROOT.rglob('*')) if p.is_file() and '__pycache__' not in p.parts} -def execute(command, cwd, out, name, mutant=False): - result = subprocess.run(command, cwd=cwd, capture_output=True, text=True, timeout=30) +def execute(command, cwd, out, name, mutant=False, *, mode, mutation=None): + timeout_seconds = 30 + started = None + clock_failure = None + try: + started = time.monotonic() + except Exception as error: + clock_failure = f'initial clock: {type(error).__name__}: {error}' + try: + result = subprocess.run(command, cwd=cwd, capture_output=True, text=True, timeout=timeout_seconds) + except subprocess.TimeoutExpired as timeout: + failures = [] + streams = {} + # TimeoutExpired streams are bytes even with text=True. In particular, + # a killed child can leave an incomplete UTF-8 sequence: never decode it. + for stream, raw in [('stdout', timeout.stdout), ('stderr', timeout.stderr)]: + streams[stream] = {'present': raw is not None, 'bytes': len(raw) if raw is not None else 0} + path = out/(name+'.'+stream) + try: + path.write_bytes(raw if raw is not None else b'') + except Exception as error: + failure = f'{path.name}: {type(error).__name__}: {error}' + failures.append(failure) + timeout.add_note('Timeout evidence persistence failed: '+failure) + elapsed = None + if clock_failure is None: + try: + measured = time.monotonic() - started + if not math.isfinite(measured) or measured < 0: + raise ValueError('elapsed time must be finite and nonnegative') + elapsed = measured + except Exception as error: + clock_failure = f'elapsed clock: {type(error).__name__}: {error}' + diagnostic_failures = [clock_failure] if clock_failure is not None else [] + if clock_failure is not None: + timeout.add_note('Timeout evidence diagnostic failed: '+clock_failure) + # Preserve the cwd argument actually supplied; filesystem resolution is + # unnecessary and can itself fail while diagnosing the original timeout. + record = {'argv': list(command), 'cwd': str(cwd), + 'run_name': name, 'mode': mode, 'mutation': mutation, 'mutant': mutant, + 'timeout_seconds': timeout_seconds, 'elapsed_seconds': elapsed, + 'status': 'timed_out', 'streams': streams, 'persistence_failures': failures, + 'diagnostic_failures': diagnostic_failures} + path = out/(name+'.timeout.json') + try: + path.write_text(json.dumps(record, sort_keys=True, indent=2, allow_nan=False)+'\n') + except Exception as error: + timeout.add_note(f'Timeout evidence persistence failed: {path.name}: {type(error).__name__}: {error}') + raise (out/(name+'.stdout')).write_text(result.stdout) (out/(name+'.stderr')).write_text(result.stderr) require(f'Ran {EXPECTED_TESTS} tests' in result.stderr and 'skipped=' not in result.stderr, 'unexpected test coverage: '+name) @@ -92,7 +141,7 @@ def main(): try: for mode, flags in [('normal', []), ('optimized', ['-O'])]: cmd = [sys.executable, '-B', *flags, '-S', '-m', 'unittest', 'discover', '-p', 'test_*.py', '-v'] - execute(cmd, ROOT, out, 'tests_'+mode) + execute(cmd, ROOT, out, 'tests_'+mode, mode=mode) result = subprocess.run([sys.executable, '-B', *flags, '-S', 'hard_gate.py'], cwd=ROOT, capture_output=True, timeout=15) require(result.returncode == 0 and result.stdout == (ROOT/'RESULTS.json').read_bytes(), 'entry/result mismatch: '+mode) (out/('output_'+mode+'.json')).write_bytes(result.stdout) @@ -103,12 +152,18 @@ def main(): scratch = out/'mutants'/mode/name shutil.copytree(ROOT, scratch, ignore=shutil.ignore_patterns('__pycache__')) (scratch/'hard_gate.py').write_text(mutated) - execute(cmd, scratch, out, 'mutation_'+mode+'_'+name, mutant=True) + execute(cmd, scratch, out, 'mutation_'+mode+'_'+name, mutant=True, mode=mode, mutation=name) report['modes'].append(mode) require(before == identities(), 'source files changed during replay') report.update(passed=True, sources_unchanged=True, source_files=before) finally: - (out/'REPORT.json').write_text(json.dumps(report, sort_keys=True, indent=2)+'\n') + active_error = sys.exc_info()[1] + try: + (out/'REPORT.json').write_text(json.dumps(report, sort_keys=True, indent=2)+'\n') + except Exception as error: + if not isinstance(active_error, subprocess.TimeoutExpired): + raise + active_error.add_note(f'Timeout evidence persistence failed: REPORT.json: {type(error).__name__}: {error}') print(json.dumps(report, sort_keys=True)) diff --git a/tests/test_downstream_validation.py b/tests/test_downstream_validation.py new file mode 100644 index 000000000..b5d42a9e9 --- /dev/null +++ b/tests/test_downstream_validation.py @@ -0,0 +1,266 @@ +"""Timeout diagnostics must retain evidence without weakening the replay gate.""" +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / 'frontiers/downstream_gate_20260925/run_validation.py' +SPEC = importlib.util.spec_from_file_location('downstream_validation', SCRIPT) +M = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(M) + + +class ValidationTimeoutTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.out = Path(self.temp.name) + self.command = [sys.executable, '-B', '-S', 'test.py'] + + def test_partial_binary_streams_survive_timeout_unchanged(self): + # Without the timeout handler, neither partial stream reaches disk. + stdout, stderr = b'partial stdout\x00\xe2\x82', b'partial stderr\xff\n' + timeout = subprocess.TimeoutExpired(self.command, 30, output=stdout, stderr=stderr) + with mock.patch.object(M.subprocess, 'run', side_effect=timeout): + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.execute(self.command, self.out, self.out, 'tests_normal', mode='normal') + self.assertIs(caught.exception, timeout) + self.assertTrue((self.out / 'tests_normal.stdout').is_file(), 'partial stdout was discarded') + self.assertEqual((self.out / 'tests_normal.stdout').read_bytes(), stdout) + self.assertEqual((self.out / 'tests_normal.stderr').read_bytes(), stderr) + + def test_timeout_metadata_has_exact_invocation_and_elapsed_time(self): + command = [sys.executable, '-B', '-O', '-S', '-m', 'unittest'] + timeout = subprocess.TimeoutExpired(command, 30, output=b'out', stderr=b'err') + with mock.patch.object(M.subprocess, 'run', side_effect=timeout) as run: + with mock.patch.object(M.time, 'monotonic', side_effect=[100.0, 130.25]): + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.execute(command, self.out, self.out, 'mutation_optimized_allow_green_ci', + mutant=True, mode='optimized', mutation='allow_green_ci') + self.assertIs(caught.exception, timeout) + self.assertEqual(run.call_args.kwargs['timeout'], 30) + self.assertTrue(run.call_args.kwargs['capture_output']) + self.assertTrue(run.call_args.kwargs['text']) + record = json.loads((self.out / 'mutation_optimized_allow_green_ci.timeout.json').read_text()) + self.assertEqual(record, { + 'argv': command, 'cwd': str(self.out), + 'run_name': 'mutation_optimized_allow_green_ci', 'mode': 'optimized', + 'mutation': 'allow_green_ci', 'mutant': True, + 'timeout_seconds': 30, 'elapsed_seconds': 30.25, 'status': 'timed_out', + 'streams': {'stdout': {'present': True, 'bytes': 3}, + 'stderr': {'present': True, 'bytes': 3}}, + 'persistence_failures': [], 'diagnostic_failures': [], + }) + + def test_none_and_empty_streams_are_distinguished(self): + for label, payload, present in [('none', None, False), ('empty', b'', True)]: + with self.subTest(label=label): + timeout = subprocess.TimeoutExpired(self.command, 30, output=payload, stderr=payload) + with mock.patch.object(M.subprocess, 'run', side_effect=timeout): + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.execute(self.command, self.out, self.out, label, mode='normal') + self.assertIs(caught.exception, timeout) + self.assertEqual((self.out / (label + '.stdout')).read_bytes(), b'') + self.assertEqual((self.out / (label + '.stderr')).read_bytes(), b'') + record = json.loads((self.out / (label + '.timeout.json')).read_text()) + for stream in ('stdout', 'stderr'): + self.assertEqual(record['streams'][stream], {'present': present, 'bytes': 0}) + self.assertIsNone(record['mutation']) + self.assertFalse(record['mutant']) + + def test_failed_stream_write_preserves_other_stream_and_original_timeout(self): + (self.out / 'broken.stdout').mkdir() # A deterministic real filesystem failure. + timeout = subprocess.TimeoutExpired(self.command, 30, output=b'out', stderr=b'err') + with mock.patch.object(M.subprocess, 'run', side_effect=timeout): + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.execute(self.command, self.out, self.out, 'broken', mode='normal') + self.assertIs(caught.exception, timeout) + self.assertEqual((self.out / 'broken.stderr').read_bytes(), b'err') + record = json.loads((self.out / 'broken.timeout.json').read_text()) + self.assertEqual(len(record['persistence_failures']), 1) + self.assertIn('broken.stdout', record['persistence_failures'][0]) + self.assertIn('IsADirectoryError', record['persistence_failures'][0]) + self.assertTrue(any('broken.stdout' in note for note in timeout.__notes__)) + + def test_failed_metadata_write_does_not_mask_original_timeout(self): + (self.out / 'broken.timeout.json').mkdir() + timeout = subprocess.TimeoutExpired(self.command, 30, output=b'out', stderr=b'err') + with mock.patch.object(M.subprocess, 'run', side_effect=timeout): + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.execute(self.command, self.out, self.out, 'broken', mode='normal') + self.assertIs(caught.exception, timeout) + self.assertEqual((self.out / 'broken.stdout').read_bytes(), b'out') + self.assertEqual((self.out / 'broken.stderr').read_bytes(), b'err') + self.assertTrue(any('broken.timeout.json' in note for note in timeout.__notes__)) + + def test_other_subprocess_errors_are_not_caught_as_timeouts(self): + error = FileNotFoundError('missing executable') + with mock.patch.object(M.subprocess, 'run', side_effect=error): + with self.assertRaises(FileNotFoundError) as caught: + M.execute(self.command, self.out, self.out, 'missing', mode='normal') + self.assertIs(caught.exception, error) + self.assertEqual(list(self.out.iterdir()), []) + + def test_completed_baseline_behavior_and_outputs_are_unchanged(self): + completed = subprocess.CompletedProcess(self.command, 0, 'baseline stdout', 'Ran 71 tests\nOK\n') + with mock.patch.object(M.subprocess, 'run', return_value=completed): + M.execute(self.command, self.out, self.out, 'baseline', mode='normal') + self.assertEqual((self.out / 'baseline.stdout').read_text(), completed.stdout) + self.assertEqual((self.out / 'baseline.stderr').read_text(), completed.stderr) + self.assertFalse((self.out / 'baseline.timeout.json').exists()) + + def test_completed_assertion_rejection_is_still_expected_only_for_mutants(self): + stderr = 'Ran 71 tests\nAssertionError: changed semantics\nFAILED (failures=1)\n' + completed = subprocess.CompletedProcess(self.command, 1, '', stderr) + with mock.patch.object(M.subprocess, 'run', return_value=completed): + M.execute(self.command, self.out, self.out, 'mutant', mutant=True, + mode='normal', mutation='allow_green_ci') + with self.assertRaisesRegex(RuntimeError, 'baseline failed'): + M.execute(self.command, self.out, self.out, 'baseline', mode='normal') + self.assertFalse((self.out / 'mutant.timeout.json').exists()) + + def test_completed_bad_coverage_or_program_errors_still_fail(self): + bad_stderr = [ + 'Ran 70 tests\nAssertionError\nFAILED (failures=1)\n', + 'Ran 71 tests\nAssertionError\nFAILED (failures=1, skipped=1)\n', + 'Ran 71 tests\nSyntaxError\nAssertionError\nFAILED (failures=1)\n', + 'Ran 71 tests\nImportError\nAssertionError\nFAILED (failures=1)\n', + 'Ran 71 tests\nERROR: module\nAssertionError\nFAILED (failures=1)\n', + 'Ran 71 tests\nFAILED (errors=1)\n', + 'Ran 71 tests\nOK\n', + ] + for stderr in bad_stderr: + with self.subTest(stderr=stderr): + completed = subprocess.CompletedProcess(self.command, 1, '', stderr) + with mock.patch.object(M.subprocess, 'run', return_value=completed): + with self.assertRaises(RuntimeError): + M.execute(self.command, self.out, self.out, 'bad', mutant=True, + mode='normal', mutation='allow_green_ci') + + def test_main_timeout_keeps_failed_report_and_stops_later_commands(self): + timeout = subprocess.TimeoutExpired(self.command, 30, output=b'partial', stderr=b'progress') + for mutant_timeout in (False, True): + with self.subTest(mutant_timeout=mutant_timeout): + out = self.out / str(mutant_timeout) + argv = [str(SCRIPT), '--output', str(out)] + baseline = subprocess.CompletedProcess(self.command, 0, '', 'Ran 71 tests\nOK\n') + gate = subprocess.CompletedProcess(self.command, 0, (M.ROOT / 'RESULTS.json').read_bytes(), b'') + responses = [baseline, gate, timeout] if mutant_timeout else [timeout] + with mock.patch.object(sys, 'argv', argv): + with mock.patch.object(M.subprocess, 'run', side_effect=responses) as run: + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.main() + self.assertIs(caught.exception, timeout) + self.assertEqual(run.call_count, len(responses)) + report = json.loads((out / 'REPORT.json').read_text()) + self.assertFalse(report['passed']) + self.assertEqual(report['modes'], []) + self.assertFalse(report['promotion_permission']) + self.assertEqual(report['distinct_tests'], 71) + self.assertEqual(report['distinct_semantic_mutations'], 24) + self.assertEqual(report['scientific_effect'], 'NONE') + name = 'mutation_normal_bypass_own_node_eligibility' if mutant_timeout else 'tests_normal' + self.assertEqual((out / (name + '.stdout')).read_bytes(), b'partial') + record = json.loads((out / (name + '.timeout.json')).read_text()) + self.assertEqual(record['mutant'], mutant_timeout) + self.assertEqual(record['mode'], 'normal') + self.assertEqual(record['mutation'], 'bypass_own_node_eligibility' if mutant_timeout else None) + self.assertFalse((out / 'tests_optimized.stdout').exists()) + + def test_failed_final_report_write_does_not_mask_timeout(self): + timeout = subprocess.TimeoutExpired(self.command, 30, output=b'out', stderr=b'err') + out = self.out / 'main' + real_write = Path.write_text + def fail_report(path, data, *args, **kwargs): + if path.name == 'REPORT.json': + raise OSError('report disk unavailable') + return real_write(path, data, *args, **kwargs) + with mock.patch.object(sys, 'argv', [str(SCRIPT), '--output', str(out)]): + with mock.patch.object(M.subprocess, 'run', side_effect=timeout): + with mock.patch.object(Path, 'write_text', fail_report): + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.main() + self.assertIs(caught.exception, timeout) + self.assertTrue(any('REPORT.json' in note for note in timeout.__notes__)) + self.assertTrue((out / 'tests_normal.timeout.json').is_file()) + self.assertFalse((out / 'REPORT.json').exists()) + + def test_clock_failures_preserve_timeout_streams_and_available_metadata(self): + clocks = [ + ('initial', [OSError('clock start unavailable')]), + ('elapsed', [100.0, OSError('clock end unavailable')]), + ('nan', [100.0, float('nan')]), + ('infinity', [100.0, float('inf')]), + ('negative', [100.0, 99.0]), + ] + for label, readings in clocks: + with self.subTest(label=label): + timeout = subprocess.TimeoutExpired(self.command, 30, output=b'out', stderr=b'err') + with mock.patch.object(M.time, 'monotonic', side_effect=readings): + with mock.patch.object(M.subprocess, 'run', side_effect=timeout) as run: + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.execute(self.command, self.out, self.out, label, mode='normal') + self.assertIs(caught.exception, timeout) + self.assertEqual(run.call_count, 1) + self.assertEqual((self.out / (label + '.stdout')).read_bytes(), b'out') + self.assertEqual((self.out / (label + '.stderr')).read_bytes(), b'err') + def reject_non_json(value): + raise ValueError(value) + record = json.loads((self.out / (label + '.timeout.json')).read_text(), + parse_constant=reject_non_json) + self.assertIsNone(record['elapsed_seconds']) + self.assertEqual(len(record['diagnostic_failures']), 1) + self.assertTrue(any('clock' in note or 'elapsed' in note for note in timeout.__notes__)) + + def test_timeout_cwd_is_the_supplied_path_without_filesystem_resolution(self): + timeout = subprocess.TimeoutExpired(self.command, 30, output=b'out', stderr=b'err') + with mock.patch.object(Path, 'resolve', side_effect=OSError('cwd unavailable')): + with mock.patch.object(M.subprocess, 'run', side_effect=timeout): + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.execute(self.command, Path('relative-cwd'), self.out, 'cwd', mode='normal') + self.assertIs(caught.exception, timeout) + self.assertEqual((self.out / 'cwd.stdout').read_bytes(), b'out') + record = json.loads((self.out / 'cwd.timeout.json').read_text()) + self.assertEqual(record['cwd'], 'relative-cwd') + + def test_initial_clock_failure_does_not_change_completed_baseline(self): + completed = subprocess.CompletedProcess(self.command, 0, 'out', 'Ran 71 tests\nOK\n') + with mock.patch.object(M.time, 'monotonic', side_effect=OSError('clock unavailable')): + with mock.patch.object(M.subprocess, 'run', return_value=completed): + M.execute(self.command, self.out, self.out, 'baseline', mode='normal') + self.assertEqual((self.out / 'baseline.stdout').read_text(), 'out') + self.assertEqual((self.out / 'baseline.stderr').read_text(), completed.stderr) + self.assertFalse((self.out / 'baseline.timeout.json').exists()) + + def test_real_child_partial_bytes_are_retained(self): + # Only the test wrapper shortens the timeout. Production still supplies 30. + real_run = subprocess.run + observed = [] + def short_run(command, **kwargs): + self.assertEqual(kwargs['timeout'], 30) + kwargs['timeout'] = 0.5 + try: + return real_run(command, **kwargs) + except subprocess.TimeoutExpired as exc: + observed.append(exc) + raise + command = [sys.executable, '-B', '-S', '-c', + "import os,time; os.write(1,b'out\\xe2\\x82'); os.write(2,b'err\\xff'); time.sleep(10)"] + with mock.patch.object(M.subprocess, 'run', side_effect=short_run): + with self.assertRaises(subprocess.TimeoutExpired) as caught: + M.execute(command, self.out, self.out, 'real', mode='normal') + self.assertIs(caught.exception, observed[0]) + self.assertEqual((self.out / 'real.stdout').read_bytes(), b'out\xe2\x82') + self.assertEqual((self.out / 'real.stderr').read_bytes(), b'err\xff') + record = json.loads((self.out / 'real.timeout.json').read_text()) + self.assertEqual(record['timeout_seconds'], 30) + self.assertGreater(record['elapsed_seconds'], 0) + + +if __name__ == '__main__': + unittest.main()