Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions .github/workflows/navigation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@ jobs:
python -B -S tools/workspace_landing_check.py > navigation-evidence/landing.txt
python -B -S -m unittest discover -s tests -p 'test_*.py' -v > navigation-evidence/tests-normal.log 2>&1
grep -Eq '^Ran [1-9][0-9]* tests? in ' navigation-evidence/tests-normal.log || { echo "::error::navigation normal discovery collected 0 tests or returned no count"; exit 1; }
python -B -O -S -m unittest discover -s tests -p 'test_*.py' -v > navigation-evidence/tests-optimized.log 2>&1
grep -Eq '^Ran [1-9][0-9]* tests? in ' navigation-evidence/tests-optimized.log || { echo "::error::navigation optimized discovery collected 0 tests or returned no count"; exit 1; }
python -B -S tools/navigation_check.py --verify-public > navigation-evidence/navigation.json
- name: Preserve actual results including failures
if: always()
Expand Down
44 changes: 13 additions & 31 deletions tests/test_navigation_discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@

ROOT = Path(__file__).resolve().parents[1]
NORMAL = ['-B', '-S', '-m', 'unittest', 'discover', '-s', 'tests', '-p', 'test_*.py', '-v']
OPTIMIZED = ['-B', '-O', '-S', '-m', 'unittest', 'discover', '-s', 'tests', '-p', 'test_*.py', '-v']
GOOD = 'Ran 2 tests in 0.001s\n\nOK\n'
ZERO = 'Ran 0 tests in 0.000s\n\nOK\n'
DRIVER = '''import json, os, sys
Expand All @@ -27,8 +26,7 @@
elif kind == 'python' and args == ['--version']:
print('inert process-result fixture; not a Python-version assertion')
elif kind == 'python' and '-m' in args:
mode = 'optimized' if '-O' in args else 'normal'
data = json.loads(Path(os.environ['FIXTURE_RESULTS']).read_text())[mode]
data = json.loads(Path(os.environ['FIXTURE_RESULTS']).read_text())['normal']
sys.stderr.write(data['output'])
raise SystemExit(data['exit'])
elif kind == 'python' and args in (
Expand Down Expand Up @@ -60,7 +58,7 @@ def setUpClass(cls):
raise RuntimeError('Bash is required for navigation shell controls')
cls.block = workflow_block()

def run_block(self, normal=GOOD, optimized=GOOD, normal_exit=0, optimized_exit=0):
def run_block(self, normal=GOOD, normal_exit=0):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
binary = root / 'bin'
Expand All @@ -75,7 +73,6 @@ def run_block(self, normal=GOOD, optimized=GOOD, normal_exit=0, optimized_exit=0
results = root / 'results.json'
results.write_text(json.dumps({
'normal': {'output': normal, 'exit': normal_exit},
'optimized': {'output': optimized, 'exit': optimized_exit},
}), encoding='utf-8')
script = root / 'run.sh'
script.write_text(self.block, encoding='utf-8')
Expand All @@ -92,65 +89,50 @@ def run_block(self, normal=GOOD, optimized=GOOD, normal_exit=0, optimized_exit=0
for path in (root / 'navigation-evidence').iterdir()}
return run, trace, logs

def assert_rejected(self, mode, output, exit_status=0):
kwargs = {mode: output, mode + '_exit': exit_status}
run, trace, logs = self.run_block(**kwargs)
def assert_rejected(self, output, exit_status=0):
run, trace, logs = self.run_block(normal=output, normal_exit=exit_status)
self.assertEqual(run.returncode, exit_status or 1, run.stdout + run.stderr)
self.assertEqual(logs['tests-' + mode + '.log'], output)
self.assertEqual(logs['tests-normal.log'], output)
self.assertNotIn(['python', '-B', '-S', 'tools/navigation_check.py', '--verify-public'], trace)
self.assertNotIn('navigation.json', logs)
self.assertEqual(trace.count(['python', *NORMAL]), 1)
self.assertEqual(trace.count(['python', *OPTIMIZED]), int(mode == 'optimized'))
if exit_status == 0:
self.assertIn('::error::navigation ' + mode + ' discovery collected 0 tests or returned no count',
self.assertIn('::error::navigation normal discovery collected 0 tests or returned no count',
run.stdout + run.stderr)

def test_positive_plural_summaries_reach_navigation(self):
run, trace, logs = self.run_block()
self.assertEqual(run.returncode, 0, run.stdout + run.stderr)
self.assertEqual(trace.count(['python', *NORMAL]), 1)
self.assertEqual(trace.count(['python', *OPTIMIZED]), 1)
self.assertEqual(logs['tests-normal.log'], GOOD)
self.assertEqual(logs['tests-optimized.log'], GOOD)
self.assertIn('navigation.json', logs)

def test_positive_singular_summaries_are_accepted(self):
singular = 'Ran 1 test in 0.001s\n\nOK\n'
run, _, logs = self.run_block(normal=singular, optimized=singular)
run, _, logs = self.run_block(normal=singular)
self.assertEqual(run.returncode, 0, run.stdout + run.stderr)
self.assertIn('navigation.json', logs)

def test_normal_zero_count_exit_zero_is_rejected(self):
self.assert_rejected('normal', ZERO)

def test_optimized_zero_count_exit_zero_is_rejected(self):
self.assert_rejected('optimized', ZERO)
self.assert_rejected(ZERO)

def test_missing_normal_summary_is_rejected(self):
self.assert_rejected('normal', 'OK\n')

def test_missing_optimized_summary_is_rejected(self):
self.assert_rejected('optimized', 'OK\n')
self.assert_rejected('OK\n')

def test_nonzero_normal_exit_is_not_masked(self):
self.assert_rejected('normal', GOOD + 'fixture failure\n', 23)

def test_nonzero_optimized_exit_is_not_masked(self):
self.assert_rejected('optimized', GOOD + 'fixture failure\n', 23)
self.assert_rejected(GOOD + 'fixture failure\n', 23)

def test_positive_skipped_count_is_not_an_empty_suite(self):
skipped = 'Ran 1 test in 0.001s\n\nOK (skipped=1)\n'
run, _, logs = self.run_block(normal=skipped, optimized=skipped)
run, _, logs = self.run_block(normal=skipped)
self.assertEqual(run.returncode, 0, run.stdout + run.stderr)
self.assertEqual(logs['tests-normal.log'], skipped)
self.assertEqual(logs['tests-optimized.log'], skipped)

def test_malformed_summary_does_not_supply_a_positive_count(self):
for output in ('prefix Ran 2 tests in 0.001s\n', 'Ran -1 tests in 0.001s\n',
'Ran 01 tests in 0.001s\n', 'Ran 2 testcases in 0.001s\n'):
for mode in ('normal', 'optimized'):
with self.subTest(output=output, mode=mode):
self.assert_rejected(mode, output)
with self.subTest(output=output):
self.assert_rejected(output)


if __name__ == '__main__':
Expand Down
Loading