From 625bfc89975efa1fd725076bc2b0961af2768279 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:05:26 -0500 Subject: [PATCH 1/3] Run both Atlas regressions in the existing sandboxed browser job Require exact two-test execution with zero skips, retain evidence, and self-test the runner. Actual hosted UI qualification remains pending. --- .github/workflows/public-shop.yml | 14 ++ docs/cbz/ATLAS_BROWSER_REGRESSIONS.md | 75 ++++++++ .../test_atlas_browser_regressions_runner.py | 177 ++++++++++++++++++ tests/test_atlas_focus_geometry.py | 3 +- tools/run_atlas_browser_regressions.py | 118 ++++++++++++ 5 files changed, 386 insertions(+), 1 deletion(-) create mode 100644 docs/cbz/ATLAS_BROWSER_REGRESSIONS.md create mode 100644 tests/test_atlas_browser_regressions_runner.py create mode 100644 tools/run_atlas_browser_regressions.py diff --git a/.github/workflows/public-shop.yml b/.github/workflows/public-shop.yml index 4bb75f288..e75bc07d7 100644 --- a/.github/workflows/public-shop.yml +++ b/.github/workflows/public-shop.yml @@ -68,6 +68,20 @@ jobs: run: python -B tools/public_shop_browser_check.py --output browser-evidence - name: Current mathematical atlas and static reading parity run: python -B tools/current_site_browser_check.py --output browser-evidence/current-atlas + - name: Exact Atlas browser runner controls + run: | + python -B -S -m unittest tests.test_atlas_browser_regressions_runner -v + python -B -O -S -m unittest tests.test_atlas_browser_regressions_runner -v + - name: Atlas focus and reader regressions in both modes + run: | + set -euo pipefail + mkdir -p browser-evidence/atlas-regressions + status=0 + python -B tools/run_atlas_browser_regressions.py --output browser-evidence/atlas-regressions/normal > browser-evidence/atlas-regressions/normal.stdout 2> browser-evidence/atlas-regressions/normal.stderr || status=1 + python -B -O tools/run_atlas_browser_regressions.py --output browser-evidence/atlas-regressions/optimized > browser-evidence/atlas-regressions/optimized.stdout 2> browser-evidence/atlas-regressions/optimized.stderr || status=1 + cat browser-evidence/atlas-regressions/*.stdout + cat browser-evidence/atlas-regressions/*.stderr >&2 + exit "$status" - name: Preserve original current-atlas report in bounded log frames if: always() timeout-minutes: 1 diff --git a/docs/cbz/ATLAS_BROWSER_REGRESSIONS.md b/docs/cbz/ATLAS_BROWSER_REGRESSIONS.md new file mode 100644 index 000000000..7a6d39f2a --- /dev/null +++ b/docs/cbz/ATLAS_BROWSER_REGRESSIONS.md @@ -0,0 +1,75 @@ +# Exact hosted Atlas browser regressions + +This engineering runner closes an execution-coverage gap for two existing UI +regressions. It does not certify mathematics, all regression coverage, or CBZ-0. +Actual browser execution on this contribution is pending hosted qualification. + +## Scope and execution + +`tools/run_atlas_browser_regressions.py` loads only the exact focus-ring and +long-reader-selection method IDs declared in `EXPECTED_IDS`. It refuses an empty, +duplicate, missing, extra or renamed inventory. Success requires exactly two +successful methods, no failures/errors/skips, no expected failures/unexpected +successes, and unchanged clean tracked source identity before/after execution. +The focus test now explicitly enables Chromium's sandbox, matching the reader +test and existing hosted browser policy. No UI assertion or bound is changed. + +The existing public-shop `browser-smoke` job supplies the hash-pinned +`tests/browser-requirements.txt` dependencies and packaged Chrome. No permission, +action pin, dependency pin, browser flag disabling security, or browser download +is introduced. Existing browser checks remain before the new runner steps. + +After installing the existing pinned requirements in that supported environment: + +``` +python -B tools/run_atlas_browser_regressions.py --output browser-evidence/atlas-regressions/normal +python -B -O tools/run_atlas_browser_regressions.py --output browser-evidence/atlas-regressions/optimized +``` + +Each output directory must be new; reuse fails without overwriting old evidence. +Each mode records a result JSON and the test-produced screenshots/measurements. +CI separately retains complete stdout/stderr, runs both modes even if one fails, +and returns failure if either fails. The existing always-upload browser artifact +preserves these files. Runtime metadata records Python/Playwright and each chosen +browser launcher path, version output and launcher-file SHA256 (a launcher may be +a wrapper, so this is not a claim to hash every underlying browser binary). +Git commit/tree and tracked cleanliness are local observations, not cryptographic +attestation or hostile-process containment. A missing dependency, denied browser +launch, assertion failure or source drift is failure; do not relax the sandbox or +reinterpret an optional skip as success. The prior local socket denial is not +retried by this contribution. + +## Self-verification + +``` +python -B -S -m unittest tests.test_atlas_browser_regressions_runner -v +python -B -O -S -m unittest tests.test_atlas_browser_regressions_runner -v +``` + +Fifteen controls run without launching a browser. They cover inventory, actual +unittest success/failure/error/subtest/skip/expected-failure/unexpected-success, +missing execution, report/exit consistency, source drift, old-output retention, +explicit sandbox parameters and workflow wiring. Six actual runner-source mutants +must fail their designated assertions with zero errors or skips: bypass inventory, +force green, ignore source drift, overwrite old evidence, return wrong exit and +retain a success report after exception. The unchanged control must pass. Finite +controls establish these contracts, not universal test-runner soundness. + +## Canonical registration addendum for the integrator + +Shared canonical files remain owned by the current baseline writer. Compose these +facts into their current versions rather than restoring the historical STATUS: + +- GOVERNANCE/TOOLING: `tools/run_atlas_browser_regressions.py`; purpose: fixed-ID, + zero-skip execution of two Atlas UI regressions; criterion: G5 execution coverage + and G11 documented tooling; usage/self-verification: exact commands above; + maintenance owner: none, self-contained; verification: 15 controls and 6 source + mutants per mode locally passed; actual hosted browser result remains pending. +- STATUS, How to verify: run both control commands, then both real runner commands + in the existing pinned browser-smoke environment; report the original result + JSON/logs and checked source identity. This does not close five separate + production-record applicability/live-source tests. +- DECISIONS: add the narrow execution-coverage runner because plain unittest can + exit zero after skipping required browser methods. Preserve all UI assertions, + source history, original failure logs and supported browser sandbox. No new UI + functionality, mathematical claim or final baseline promotion is introduced. diff --git a/tests/test_atlas_browser_regressions_runner.py b/tests/test_atlas_browser_regressions_runner.py new file mode 100644 index 000000000..cf03b4607 --- /dev/null +++ b/tests/test_atlas_browser_regressions_runner.py @@ -0,0 +1,177 @@ +"""Controls for exact two-test execution; these never launch a browser.""" +import ast +import importlib.util +import io +import json +import tempfile +from unittest import mock +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[1] +PATH = ROOT / 'tools/run_atlas_browser_regressions.py' + + +def load_runner(): + spec = importlib.util.spec_from_file_location('atlas_browser_runner', PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class AtlasBrowserRunnerControls(unittest.TestCase): + def setUp(self): + self.runner = load_runner() + + def suite(self, behavior='pass', ids=None): + ids = self.runner.EXPECTED_IDS if ids is None else ids + cases = [] + for index, identity in enumerate(ids): + def test(case, selected=index == 0): + if not selected: + return + if behavior == 'skip': + case.skipTest('synthetic missing prerequisite') + elif behavior in ('failure', 'expected_failure'): + case.fail('synthetic assertion failure') + elif behavior == 'error': + raise RuntimeError('synthetic runtime error') + elif behavior == 'subtest_failure': + with case.subTest(value=1): + case.fail('synthetic subtest failure') + if behavior in ('expected_failure', 'unexpected_success') and index == 0: + test = unittest.expectedFailure(test) + cls = type('SyntheticCase', (unittest.TestCase,), { + 'runTest': test, 'id': lambda case, value=identity: value}) + cases.append(cls()) + return unittest.TestSuite(cases) + + def test_exact_two_successes_pass(self): + result = self.runner.run_suite(self.suite(), io.StringIO()) + self.assertTrue(result['passed']) + self.assertEqual(result['tests_run'], 2) + self.assertEqual(result['successful_ids'], list(self.runner.EXPECTED_IDS)) + + def test_empty_inventory_refused(self): + with self.assertRaisesRegex(ValueError, 'test inventory'): + self.runner.run_suite(self.suite(ids=[]), io.StringIO()) + + def test_duplicate_inventory_refused(self): + with self.assertRaisesRegex(ValueError, 'test inventory'): + self.runner.run_suite(self.suite(ids=[self.runner.EXPECTED_IDS[0]] * 2), io.StringIO()) + + def test_wrong_and_extra_ids_refused(self): + for ids in [('wrong', self.runner.EXPECTED_IDS[1]), (*self.runner.EXPECTED_IDS, 'extra')]: + with self.subTest(ids=ids), self.assertRaisesRegex(ValueError, 'test inventory'): + self.runner.run_suite(self.suite(ids=ids), io.StringIO()) + + def test_skip_is_failure(self): + result = self.runner.run_suite(self.suite('skip'), io.StringIO()) + self.assertFalse(result['passed']) + self.assertEqual(len(result['skips']), 1) + + def test_failures_errors_and_subtests_are_not_success(self): + for kind in ('failure', 'error', 'subtest_failure'): + with self.subTest(kind=kind): + self.assertFalse(self.runner.run_suite(self.suite(kind), io.StringIO())['passed']) + + def test_expected_failure_is_not_success(self): + result = self.runner.run_suite(self.suite('expected_failure'), io.StringIO()) + self.assertFalse(result['passed']) + self.assertEqual(result['expected_failures'], 1) + + def test_unexpected_success_is_not_success(self): + result = self.runner.run_suite(self.suite('unexpected_success'), io.StringIO()) + self.assertFalse(result['passed']) + self.assertEqual(result['unexpected_successes'], 1) + + def test_missing_execution_is_not_success(self): + class NoExecution(unittest.TestSuite): + def run(self, result, debug=False): + return result + suite = NoExecution(list(self.suite())) + self.assertFalse(self.runner.run_suite(suite, io.StringIO())['passed']) + + def invoke_main(self, directory, behavior='pass', identities=None): + identities = identities or [{'commit': 'a', 'tree': 'b'}] * 2 + with mock.patch.object(self.runner, 'git_identity', side_effect=identities), \ + mock.patch.object(self.runner, 'runtime_identity', return_value={'synthetic': True}), \ + mock.patch.object(self.runner.unittest.defaultTestLoader, 'loadTestsFromNames', return_value=self.suite(behavior)), \ + mock.patch.dict(self.runner.os.environ), \ + mock.patch.object(self.runner.sys, 'path', list(self.runner.sys.path)), \ + mock.patch.object(self.runner.sys, 'stderr', io.StringIO()): + return self.runner.main(['--output', str(directory)]) + + def test_main_success_and_failure_exit_match_new_report(self): + for behavior, expected in [('pass', 0), ('skip', 1), ('error', 1), ('expected_failure', 1)]: + with self.subTest(behavior=behavior), tempfile.TemporaryDirectory() as temp: + output = Path(temp) / 'attempt' + self.assertEqual(self.invoke_main(output, behavior), expected) + report = json.loads((output / 'result.json').read_text()) + self.assertEqual(report['passed'], expected == 0) + self.assertEqual(report['scientific_effect'], 'NONE') + + def test_reused_output_is_refused_without_overwrite(self): + with tempfile.TemporaryDirectory() as temp: + output = Path(temp) + previous = b'original retained bytes\n' + (output / 'result.json').write_bytes(previous) + with self.assertRaises(FileExistsError): + self.invoke_main(output) + self.assertEqual((output / 'result.json').read_bytes(), previous) + + def test_source_drift_cannot_leave_successful_report(self): + with tempfile.TemporaryDirectory() as temp: + output = Path(temp) / 'attempt' + identities = [{'commit': 'a', 'tree': 'b'}, {'commit': 'c', 'tree': 'd'}] + self.assertEqual(self.invoke_main(output, identities=identities), 1) + report = json.loads((output / 'result.json').read_text()) + self.assertFalse(report['passed']) + self.assertIn('source identity changed', report['error']) + + def test_six_runner_source_mutants_are_killed_by_assertions(self): + source = PATH.read_text() + mutations = ( + ('inventory', 'if tuple(test_ids(suite)) != EXPECTED_IDS:', 'if False:', 'test_empty_inventory_refused'), + ('green', "'passed': passed", "'passed': True", 'test_skip_is_failure'), + ('drift', "if report['source_before'] != report['source_after']:", 'if False:', 'test_source_drift_cannot_leave_successful_report'), + ('overwrite', 'output.mkdir(parents=True, exist_ok=False)', 'output.mkdir(parents=True, exist_ok=True)', 'test_reused_output_is_refused_without_overwrite'), + ('exit', "return 0 if report['passed'] else 1", 'return 0', 'test_main_success_and_failure_exit_match_new_report'), + ('error', "report['passed'] = False", 'pass', 'test_source_drift_cannot_leave_successful_report'), + ) + for name, before, after, method in mutations: + with self.subTest(mutation=name), tempfile.TemporaryDirectory() as temp: + self.assertIn(before, source) + path = Path(temp) / 'mutant.py' + path.write_text(source.replace(before, after, 1)) + spec = importlib.util.spec_from_file_location('mutant_' + name, path) + mutant = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mutant) + with mock.patch(__name__ + '.load_runner', return_value=mutant): + result = unittest.TextTestRunner(stream=io.StringIO()).run( + unittest.TestSuite([type(self)(method)])) + self.assertGreater(len(result.failures), 0, name) + self.assertEqual(result.errors, [], name) + self.assertEqual(result.skipped, [], name) + + def test_focus_and_reader_explicitly_enable_sandbox(self): + for name in ('test_atlas_focus_geometry.py', 'test_atlas_reader_scroll.py'): + tree = ast.parse((ROOT / 'tests' / name).read_text()) + launches = [node for node in ast.walk(tree) if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) and node.func.attr == 'launch'] + self.assertEqual(len(launches), 1) + values = {item.arg: item.value for item in launches[0].keywords} + self.assertIn('chromium_sandbox', values) + self.assertIs(values['chromium_sandbox'].value, True) + + def test_workflow_runs_both_modes_and_retains_evidence(self): + source = (ROOT / '.github/workflows/public-shop.yml').read_text() + self.assertIn('python -B tools/run_atlas_browser_regressions.py --output browser-evidence/atlas-regressions/normal', source) + self.assertIn('python -B -O tools/run_atlas_browser_regressions.py --output browser-evidence/atlas-regressions/optimized', source) + self.assertIn('tests.test_atlas_browser_regressions_runner', source) + self.assertIn('path: browser-evidence', source) + self.assertIn('if: always()', source) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_atlas_focus_geometry.py b/tests/test_atlas_focus_geometry.py index a7409191e..02ef2c69c 100644 --- a/tests/test_atlas_focus_geometry.py +++ b/tests/test_atlas_focus_geometry.py @@ -31,7 +31,8 @@ def log_message(self, *_args): thread.start() try: with sync_playwright() as playwright: - browser = playwright.chromium.launch(executable_path=CHROMIUM, headless=True) + browser = playwright.chromium.launch(executable_path=CHROMIUM, headless=True, + chromium_sandbox=True) try: page = browser.new_page(viewport={"width": 390, "height": 844}, reduced_motion="reduce") diff --git a/tools/run_atlas_browser_regressions.py b/tools/run_atlas_browser_regressions.py new file mode 100644 index 000000000..3bf3a61ae --- /dev/null +++ b/tools/run_atlas_browser_regressions.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Run exactly two Atlas browser tests; skips are failure, never optional green. + +Use without -S after installing tests/browser-requirements.txt. The output +folder must be new. This is trusted unittest execution, not hostile-code +containment, mathematical certification or a signature of remote execution. +""" +import argparse +import hashlib +import importlib.metadata +import json +import os +from pathlib import Path +import subprocess +import sys +import traceback +import unittest + +ROOT = Path(__file__).resolve().parents[1] +EXPECTED_IDS = ( + 'tests.test_atlas_focus_geometry.AtlasListFocusGeometry.test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary', + 'tests.test_atlas_reader_scroll.AtlasReaderScroll.test_new_selection_after_a_long_read_keeps_its_heading_visible', +) + + +def test_ids(suite): + for item in suite: + if isinstance(item, unittest.TestSuite): + yield from test_ids(item) + else: + yield item.id() + + +class ExactResult(unittest.TextTestResult): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.successful_ids = [] + + def addSuccess(self, test): + self.successful_ids.append(test.id()) + super().addSuccess(test) + + +def run_suite(suite, stream): + if tuple(test_ids(suite)) != EXPECTED_IDS: + raise ValueError('test inventory must equal the two fixed Atlas method IDs') + result = unittest.TextTestRunner(stream=stream, verbosity=2, + resultclass=ExactResult).run(suite) + passed = (result.wasSuccessful() and result.testsRun == 2 + and tuple(result.successful_ids) == EXPECTED_IDS + and not result.skipped and not result.expectedFailures + and not result.unexpectedSuccesses) + return {'passed': passed, 'tests_run': result.testsRun, + 'successful_ids': result.successful_ids, + 'skips': [[test.id(), reason] for test, reason in result.skipped], + 'failures': len(result.failures), 'errors': len(result.errors), + 'expected_failures': len(result.expectedFailures), + 'unexpected_successes': len(result.unexpectedSuccesses)} + + +def git_identity(): + def git(*args): + return subprocess.check_output(['git', *args], cwd=ROOT, text=True).strip() + if git('status', '--porcelain', '--untracked-files=no'): + raise ValueError('tracked source is dirty') + return {'commit': git('rev-parse', 'HEAD'), 'tree': git('rev-parse', 'HEAD^{tree}')} + + +def runtime_identity(): + browsers = [] + for module_name, attribute in ( + ('tests.test_atlas_focus_geometry', 'CHROMIUM'), + ('tests.test_atlas_reader_scroll', 'BROWSER')): + executable = getattr(sys.modules[module_name], attribute) + if not executable: + raise ValueError('required packaged browser executable is unavailable') + path = Path(executable).resolve(strict=True) + version = subprocess.check_output([str(path), '--version'], text=True, + stderr=subprocess.STDOUT, timeout=15).strip() + browsers.append({'module': module_name, 'path': str(path), 'version': version, + 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}) + return {'python': sys.version, 'optimize': sys.flags.optimize, + 'playwright': importlib.metadata.version('playwright'), 'browsers': browsers} + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, required=True) + output = parser.parse_args(argv).output.resolve() + # Refuse reuse before any browser/test work; retain earlier attempts intact. + output.mkdir(parents=True, exist_ok=False) + report = {'schema': 'atlas-browser-regressions/1', 'passed': False, + 'expected_ids': list(EXPECTED_IDS), 'scientific_effect': 'NONE'} + try: + report['source_before'] = git_identity() + sys.path.insert(0, str(ROOT)) + os.environ['ATLAS_FOCUS_EVIDENCE'] = str(output / 'focus') + os.environ['ATLAS_READER_EVIDENCE'] = str(output / 'reader') + suite = unittest.defaultTestLoader.loadTestsFromNames(EXPECTED_IDS) + # Loader errors and absent prerequisites must not become a skip-green. + if tuple(test_ids(suite)) != EXPECTED_IDS: + raise ValueError('test inventory must equal the two fixed Atlas method IDs') + report['runtime'] = runtime_identity() + report.update(run_suite(suite, sys.stderr)) + report['source_after'] = git_identity() + if report['source_before'] != report['source_after']: + raise ValueError('source identity changed during browser tests') + except Exception as error: + report['passed'] = False + report['error'] = f'{type(error).__name__}: {error}' + traceback.print_exc() + finally: + (output / 'result.json').write_text(json.dumps(report, indent=2, sort_keys=True) + '\n') + return 0 if report['passed'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) From 8d24c37ff52c012294d71bbcbf08bd24fae4e22a Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:26:36 -0500 Subject: [PATCH 2/3] Authenticate browser mutation kills by actual test-body assertion origin --- docs/cbz/ATLAS_BROWSER_REGRESSIONS.md | 42 ++- .../browser-origin-20261029/README.md | 24 ++ .../check_grader_mutations.py | 27 ++ .../mutations-normal.json | 48 ++++ .../mutations-normal.stderr | 3 + .../mutations-optimized.json | 48 ++++ .../mutations-optimized.stderr | 3 + .../browser-origin-20261029/normal.log | 50 ++++ .../browser-origin-20261029/optimized.log | 50 ++++ .../original-false-accept-normal.log | 2 + .../original-false-accept-optimized.log | 2 + .../browser-origin-20261029/preservation.json | 15 ++ .../test_atlas_browser_regressions_runner.py | 239 +++++++++++++++++- 13 files changed, 548 insertions(+), 5 deletions(-) create mode 100644 docs/cbz/evidence/browser-origin-20261029/README.md create mode 100644 docs/cbz/evidence/browser-origin-20261029/check_grader_mutations.py create mode 100644 docs/cbz/evidence/browser-origin-20261029/mutations-normal.json create mode 100644 docs/cbz/evidence/browser-origin-20261029/mutations-normal.stderr create mode 100644 docs/cbz/evidence/browser-origin-20261029/mutations-optimized.json create mode 100644 docs/cbz/evidence/browser-origin-20261029/mutations-optimized.stderr create mode 100644 docs/cbz/evidence/browser-origin-20261029/normal.log create mode 100644 docs/cbz/evidence/browser-origin-20261029/optimized.log create mode 100644 docs/cbz/evidence/browser-origin-20261029/original-false-accept-normal.log create mode 100644 docs/cbz/evidence/browser-origin-20261029/original-false-accept-optimized.log create mode 100644 docs/cbz/evidence/browser-origin-20261029/preservation.json diff --git a/docs/cbz/ATLAS_BROWSER_REGRESSIONS.md b/docs/cbz/ATLAS_BROWSER_REGRESSIONS.md index 47d1c8819..e7b019b57 100644 --- a/docs/cbz/ATLAS_BROWSER_REGRESSIONS.md +++ b/docs/cbz/ATLAS_BROWSER_REGRESSIONS.md @@ -54,7 +54,7 @@ python -B -S -m unittest tests.test_atlas_browser_regressions_runner -v python -B -O -S -m unittest tests.test_atlas_browser_regressions_runner -v ``` -Seventeen controls run without launching a browser. They cover inventory, actual +Thirty controls run without launching a browser. The original seventeen are retained; thirteen additional controls qualify the mutation grader’s lifecycle and assertion-origin checks. They cover inventory, actual unittest success/failure/error/subtest/skip/expected-failure/unexpected-success, missing execution, report/exit consistency, source drift, old-output retention, explicit sandbox parameters, workflow wiring, ten launcher-selection cases and @@ -73,7 +73,7 @@ facts into their current versions rather than restoring the historical STATUS: - GOVERNANCE/TOOLING: `tools/run_atlas_browser_regressions.py`; purpose: fixed-ID, zero-skip execution of two Atlas UI regressions; criterion: G5 execution coverage and G11 documented tooling; usage/self-verification: exact commands above; - maintenance owner: none, self-contained; verification: 17 runner controls (32 combined browser-contract controls) and 6 + maintenance owner: none, self-contained; historical verification: 17 runner controls (32 combined browser-contract controls) and 6 runner-source mutants per mode locally passed in the fresh recovery checkout; actual hosted execution passed on the historical corrected tree below, with new composition qualification still pending. - STATUS, How to verify: run both control commands, then both real runner commands @@ -109,3 +109,41 @@ The original focus failure remains in its result and stderr records. These are engineering observations only, not mathematical evidence or final G5 completion. The later composition onto Main `1e2be44c69e08e47472041ac329e1b27f7dec793` preserves this evidence as historical; it still needs a new hosted execution. + + +## 2026-10-29 assertion-origin amendment + +At parent tree `6786ec405cef50e0fef73c7fcd9ca3945c555230`, the six-mutant +meta-test accepted setup-only AssertionErrors as kills. An independent normal and +optimized reproduction is retained in `evidence/browser-origin-20261029/`. +This demonstrates a future-use grading defect; it does not establish that any +historical finite kill or actual browser result was false. + +The test-only grader now captures actual exception/code identity before unittest +formats/clears traceback data. Admission requires the selected synchronous body +exactly once, the complete terminal population, no fixture/error/skip/expected +failure, and only the selected body plus explicitly trusted unittest assertion +helper code on the escaping failure path. Subject, helper, callback, forged-file, +setup, teardown and cleanup assertions are refused. Deferred/non-None body returns +and missing/duplicate terminal success events are refused. Each production-source +mutant first runs its unmodified designated baseline. All original assertions, +production runner/browser sources, workflow bytes and sandbox policy are unchanged. + +Current scoped controls: 30 methods per mode, or 45 when combined with the two +existing browser-contract modules; six original production mutants per mode. +Six additional grader safeguard mutations are checked by an unchanged trusted +copy of the repaired raw-origin classifier, not by their own mutated classifier: + +``` +python -B docs/cbz/evidence/browser-origin-20261029/check_grader_mutations.py +python -B -O docs/cbz/evidence/browser-origin-20261029/check_grader_mutations.py +``` + +The deferred-body mutation deliberately removes both static and runtime refusal; +its unawaited-coroutine warning is retained in stderr as an adverse observation, +not accepted as a kill. The required kill is a genuine self-test assertion. +These are finite engineering controls in a trusted interpreter with frozen tests; +this is not hostile-process containment or an exhaustive proof of unittest. +New composed-tree full regression/actual-browser execution remains pending. +Canonical tooling registration should include this amendment and its self-tests, +not relabel historical source receipts as current-source qualification. diff --git a/docs/cbz/evidence/browser-origin-20261029/README.md b/docs/cbz/evidence/browser-origin-20261029/README.md new file mode 100644 index 000000000..ccfb972a7 --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/README.md @@ -0,0 +1,24 @@ +# Browser mutation-grader origin correction + +Scoped engineering evidence, 2026-10-29. Original false acceptance was reproduced +against source tree6786ec405cef50e0fef73c7fcd9ca3945c555230. Monkeypatching only +the child class setUp to raise AssertionError made the old six-mutant meta-test +accept every failed setup as a kill. The actual test bodies never ran. Both +normal and optimized observations are retained. DD lines in those original +combined logs refer to the separately repaired Math541 harness, not browser scope. + +Final local commands and exact limits are documented in +`docs/cbz/ATLAS_BROWSER_REGRESSIONS.md`. All45 combined browser-contract controls +passed in both modes, including30 controls in the amended module, original six +production source mutants and13 new lifecycle/origin controls. Six additional +grader guard mutations per mode were graded with a pristine independent copy +of the repaired structured classifier, not the mutant's own admission flags. +Raw stdout/stderr are preserved; warning in the lazy-body adverse mutant is not +a passing oracle. Full repository and actual browser execution on this source +remain unexecuted here. Production runner/browser/workflow files are unchanged. + +Parent review independently reran30methods normally and optimized, checked the +raw-origin classifier and pristine meta-runner, and required explicit target +cardinality refusal under -O and portable repository-relative paths. Both were +corrected before final rerun. This is same-family engineering review, not a +formal PR approval, mathematical proof, universal harness soundness or CBZ-0. diff --git a/docs/cbz/evidence/browser-origin-20261029/check_grader_mutations.py b/docs/cbz/evidence/browser-origin-20261029/check_grader_mutations.py new file mode 100644 index 000000000..5fcfe88c7 --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/check_grader_mutations.py @@ -0,0 +1,27 @@ +import importlib.util,sys,json,tempfile +from pathlib import Path +ROOT=Path(__file__).resolve().parents[4] +PATH=ROOT/'tests/test_atlas_browser_regressions_runner.py' +def load(name,path): + spec=importlib.util.spec_from_file_location(name,path);m=importlib.util.module_from_spec(spec);sys.modules[name]=m;spec.loader.exec_module(m);return m +trusted=load('trusted_browser_origin',PATH);source=PATH.read_text();rows=[] +cases=[ + ('origin_frames','and all(code in _ASSERTION_CODES for code in codes[positions[0]+1:])','and True','test_subject_helper_callback_assertions_cannot_kill'), + ('phase',"and state.get('phase') in ('body', 'body-exited')",'and True','test_later_phase_calling_selected_body_cannot_kill'), + ('origin_admission','and all(v for _, v in result.origins)','and True','test_subject_helper_callback_assertions_cannot_kill'), + ('lazy_bodies',' or inspect.iscoroutinefunction(selected) or inspect.isgeneratorfunction(selected) or inspect.isasyncgenfunction(selected)','', 'test_deferred_bodies_refused'), + ('baseline','result.oracle_baseline = complete and result.wasSuccessful() and not result.origins','result.oracle_baseline = True','test_error_skip_expected_failure_refused'), + ('terminal_accounting','complete = (complete and accounted == set(expected)\n and len(result.successful) == len(set(result.successful))\n and not (set(result.successful) & {identity for identity, unused in result.origins}))','complete = complete','test_missing_or_duplicate_terminal_success_refused'), +] +for name,before,after,method in cases: + if source.count(before) != 1: raise RuntimeError('nonunique mutation target: '+name) + baseline=trusted.run_oracle_controls(trusted.OracleOriginControls,(method,)) + if not baseline.oracle_baseline:raise RuntimeError('bad baseline '+name) + with tempfile.TemporaryDirectory() as tmp: + p=Path(tmp)/'mutant.py';changed=source.replace(before,after,1) + if name=='lazy_bodies': changed=changed.replace('if returned is not None:', 'if False:') + p.write_text(changed);mutant=load('browser_origin_'+name,p) + result=trusted.run_oracle_controls(mutant.OracleOriginControls,(method,)) + if not result.oracle_killed:raise RuntimeError('not intended assertion kill '+name+repr(result.failures)+repr(result.errors)) + rows.append({'name':name,'method':method,'intended_assertion_kill':True,'failures':len(result.failures),'errors':len(result.errors)}) +print(json.dumps({'optimized':sys.flags.optimize,'source_sha256':__import__('hashlib').sha256(source.encode()).hexdigest(),'results':rows},indent=2)) diff --git a/docs/cbz/evidence/browser-origin-20261029/mutations-normal.json b/docs/cbz/evidence/browser-origin-20261029/mutations-normal.json new file mode 100644 index 000000000..64f27983f --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/mutations-normal.json @@ -0,0 +1,48 @@ +{ + "optimized": 0, + "source_sha256": "2d2b2d477f632cf86096d623cf6b29616c52e3cf89328b280f995acdeeefb0c6", + "results": [ + { + "name": "origin_frames", + "method": "test_subject_helper_callback_assertions_cannot_kill", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "phase", + "method": "test_later_phase_calling_selected_body_cannot_kill", + "intended_assertion_kill": true, + "failures": 1, + "errors": 0 + }, + { + "name": "origin_admission", + "method": "test_subject_helper_callback_assertions_cannot_kill", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "lazy_bodies", + "method": "test_deferred_bodies_refused", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "baseline", + "method": "test_error_skip_expected_failure_refused", + "intended_assertion_kill": true, + "failures": 4, + "errors": 0 + }, + { + "name": "terminal_accounting", + "method": "test_missing_or_duplicate_terminal_success_refused", + "intended_assertion_kill": true, + "failures": 2, + "errors": 0 + } + ] +} diff --git a/docs/cbz/evidence/browser-origin-20261029/mutations-normal.stderr b/docs/cbz/evidence/browser-origin-20261029/mutations-normal.stderr new file mode 100644 index 000000000..d099706a2 --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/mutations-normal.stderr @@ -0,0 +1,3 @@ +/opt/codex/runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/unittest/case.py:634: RuntimeWarning: coroutine 'OracleOriginControls.test_deferred_bodies_refused..coroutine' was never awaited + self._callTestMethod(testMethod) +RuntimeWarning: Enable tracemalloc to get the object allocation traceback diff --git a/docs/cbz/evidence/browser-origin-20261029/mutations-optimized.json b/docs/cbz/evidence/browser-origin-20261029/mutations-optimized.json new file mode 100644 index 000000000..c51a0e2bd --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/mutations-optimized.json @@ -0,0 +1,48 @@ +{ + "optimized": 1, + "source_sha256": "2d2b2d477f632cf86096d623cf6b29616c52e3cf89328b280f995acdeeefb0c6", + "results": [ + { + "name": "origin_frames", + "method": "test_subject_helper_callback_assertions_cannot_kill", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "phase", + "method": "test_later_phase_calling_selected_body_cannot_kill", + "intended_assertion_kill": true, + "failures": 1, + "errors": 0 + }, + { + "name": "origin_admission", + "method": "test_subject_helper_callback_assertions_cannot_kill", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "lazy_bodies", + "method": "test_deferred_bodies_refused", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "baseline", + "method": "test_error_skip_expected_failure_refused", + "intended_assertion_kill": true, + "failures": 4, + "errors": 0 + }, + { + "name": "terminal_accounting", + "method": "test_missing_or_duplicate_terminal_success_refused", + "intended_assertion_kill": true, + "failures": 2, + "errors": 0 + } + ] +} diff --git a/docs/cbz/evidence/browser-origin-20261029/mutations-optimized.stderr b/docs/cbz/evidence/browser-origin-20261029/mutations-optimized.stderr new file mode 100644 index 000000000..d099706a2 --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/mutations-optimized.stderr @@ -0,0 +1,3 @@ +/opt/codex/runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/unittest/case.py:634: RuntimeWarning: coroutine 'OracleOriginControls.test_deferred_bodies_refused..coroutine' was never awaited + self._callTestMethod(testMethod) +RuntimeWarning: Enable tracemalloc to get the object allocation traceback diff --git a/docs/cbz/evidence/browser-origin-20261029/normal.log b/docs/cbz/evidence/browser-origin-20261029/normal.log new file mode 100644 index 000000000..6e225ad0f --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/normal.log @@ -0,0 +1,50 @@ +test_both_tests_select_override_then_chrome_then_chromium (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_both_tests_select_override_then_chrome_then_chromium) ... ok +test_duplicate_inventory_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_duplicate_inventory_refused) ... ok +test_empty_inventory_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_empty_inventory_refused) ... ok +test_exact_two_successes_pass (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_exact_two_successes_pass) ... ok +test_expected_failure_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_expected_failure_is_not_success) ... ok +test_failures_errors_and_subtests_are_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_failures_errors_and_subtests_are_not_success) ... ok +test_focus_and_reader_explicitly_enable_sandbox (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_focus_and_reader_explicitly_enable_sandbox) ... ok +test_main_success_and_failure_exit_match_new_report (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_main_success_and_failure_exit_match_new_report) ... ok +test_missing_execution_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_missing_execution_is_not_success) ... ok +test_reused_output_is_refused_without_overwrite (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_reused_output_is_refused_without_overwrite) ... ok +test_runtime_identity_records_each_test_selected_launcher (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_runtime_identity_records_each_test_selected_launcher) ... ok +test_six_runner_source_mutants_are_killed_by_assertions (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_six_runner_source_mutants_are_killed_by_assertions) ... ok +test_skip_is_failure (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_skip_is_failure) ... ok +test_source_drift_cannot_leave_successful_report (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_source_drift_cannot_leave_successful_report) ... ok +test_unexpected_success_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_unexpected_success_is_not_success) ... ok +test_workflow_runs_both_modes_and_retains_evidence (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_workflow_runs_both_modes_and_retains_evidence) ... ok +test_wrong_and_extra_ids_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_wrong_and_extra_ids_refused) ... ok +test_cleanup_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_cleanup_assertions_cannot_kill) ... ok +test_deferred_bodies_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_deferred_bodies_refused) ... ok +test_empty_duplicate_population_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_empty_duplicate_population_refused) ... ok +test_error_skip_expected_failure_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_error_skip_expected_failure_refused) ... ok +test_fixture_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_fixture_assertions_cannot_kill) ... ok +test_forged_unittest_filename_is_not_authority (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_forged_unittest_filename_is_not_authority) ... ok +test_genuine_baseline_and_assertion_origins (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_genuine_baseline_and_assertion_origins) ... ok +test_later_phase_calling_selected_body_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_later_phase_calling_selected_body_cannot_kill) ... ok +test_missing_or_duplicate_terminal_success_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_missing_or_duplicate_terminal_success_refused) ... ok +test_repeated_body_execution_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_repeated_body_execution_refused) ... ok +test_setup_calling_body_does_not_count_as_execution (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_setup_calling_body_does_not_count_as_execution) ... ok +test_subject_helper_callback_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_subject_helper_callback_assertions_cannot_kill) ... ok +test_synchronous_returning_deferred_or_value_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_synchronous_returning_deferred_or_value_refused) ... ok +test_missing_and_malformed_source_write_failure_report (tests.test_current_site_browser_contract.BrowserReportFailureTests.test_missing_and_malformed_source_write_failure_report) ... ok +test_published_overlay_and_snapshot_only_static_reading_have_separate_exact_expectations (tests.test_current_site_browser_contract.ResumeContractTests.test_published_overlay_and_snapshot_only_static_reading_have_separate_exact_expectations) ... ok +test_wrong_shared_fields_phantom_links_and_false_operational_state_are_refused (tests.test_current_site_browser_contract.ResumeContractTests.test_wrong_shared_fields_phantom_links_and_false_operational_state_are_refused) ... ok +test_browser_dependencies_are_separate_and_version_pinned (tests.test_public_shop_browser_contract.BrowserCheckContract.test_browser_dependencies_are_separate_and_version_pinned) ... ok +test_browser_job_is_read_only_and_retains_failure_artifacts (tests.test_public_shop_browser_contract.BrowserCheckContract.test_browser_job_is_read_only_and_retains_failure_artifacts) ... ok +test_runner_is_loopback_only_with_a_four_case_matrix (tests.test_public_shop_browser_contract.BrowserCheckContract.test_runner_is_loopback_only_with_a_four_case_matrix) ... ok +test_bibtex_browser_checks_compare_real_clipboard_and_keep_failure_paths (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_bibtex_browser_checks_compare_real_clipboard_and_keep_failure_paths) ... ok +test_bibtex_disabled_checks_keep_the_hidden_button_addressable (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_bibtex_disabled_checks_keep_the_hidden_button_addressable) ... ok +test_clipboard_denial_fixture_reinitializes_after_same_document_history (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_clipboard_denial_fixture_reinitializes_after_same_document_history) ... ok +test_existing_runner_covers_source_quote_and_keyboard_disclosure (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_existing_runner_covers_source_quote_and_keyboard_disclosure) ... ok +test_new_reader_controls_are_reached_by_the_existing_browser_flows (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_new_reader_controls_are_reached_by_the_existing_browser_flows) ... ok +test_other_teaching_exports_are_reached_in_the_four_source_cases (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_other_teaching_exports_are_reached_in_the_four_source_cases) ... ok +test_pin_source_enumeration_preserves_the_downloaded_artifact_path (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_pin_source_enumeration_preserves_the_downloaded_artifact_path) ... ok +test_reader_tools_have_desktop_and_mobile_browser_evidence (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_reader_tools_have_desktop_and_mobile_browser_evidence) ... ok +test_rendered_layout_cases_read_what_the_browser_drew (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_rendered_layout_cases_read_what_the_browser_drew) ... ok + +---------------------------------------------------------------------- +Ran 45 tests in 0.611s + +OK diff --git a/docs/cbz/evidence/browser-origin-20261029/optimized.log b/docs/cbz/evidence/browser-origin-20261029/optimized.log new file mode 100644 index 000000000..363372752 --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/optimized.log @@ -0,0 +1,50 @@ +test_both_tests_select_override_then_chrome_then_chromium (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_both_tests_select_override_then_chrome_then_chromium) ... ok +test_duplicate_inventory_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_duplicate_inventory_refused) ... ok +test_empty_inventory_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_empty_inventory_refused) ... ok +test_exact_two_successes_pass (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_exact_two_successes_pass) ... ok +test_expected_failure_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_expected_failure_is_not_success) ... ok +test_failures_errors_and_subtests_are_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_failures_errors_and_subtests_are_not_success) ... ok +test_focus_and_reader_explicitly_enable_sandbox (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_focus_and_reader_explicitly_enable_sandbox) ... ok +test_main_success_and_failure_exit_match_new_report (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_main_success_and_failure_exit_match_new_report) ... ok +test_missing_execution_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_missing_execution_is_not_success) ... ok +test_reused_output_is_refused_without_overwrite (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_reused_output_is_refused_without_overwrite) ... ok +test_runtime_identity_records_each_test_selected_launcher (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_runtime_identity_records_each_test_selected_launcher) ... ok +test_six_runner_source_mutants_are_killed_by_assertions (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_six_runner_source_mutants_are_killed_by_assertions) ... ok +test_skip_is_failure (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_skip_is_failure) ... ok +test_source_drift_cannot_leave_successful_report (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_source_drift_cannot_leave_successful_report) ... ok +test_unexpected_success_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_unexpected_success_is_not_success) ... ok +test_workflow_runs_both_modes_and_retains_evidence (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_workflow_runs_both_modes_and_retains_evidence) ... ok +test_wrong_and_extra_ids_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_wrong_and_extra_ids_refused) ... ok +test_cleanup_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_cleanup_assertions_cannot_kill) ... ok +test_deferred_bodies_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_deferred_bodies_refused) ... ok +test_empty_duplicate_population_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_empty_duplicate_population_refused) ... ok +test_error_skip_expected_failure_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_error_skip_expected_failure_refused) ... ok +test_fixture_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_fixture_assertions_cannot_kill) ... ok +test_forged_unittest_filename_is_not_authority (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_forged_unittest_filename_is_not_authority) ... ok +test_genuine_baseline_and_assertion_origins (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_genuine_baseline_and_assertion_origins) ... ok +test_later_phase_calling_selected_body_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_later_phase_calling_selected_body_cannot_kill) ... ok +test_missing_or_duplicate_terminal_success_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_missing_or_duplicate_terminal_success_refused) ... ok +test_repeated_body_execution_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_repeated_body_execution_refused) ... ok +test_setup_calling_body_does_not_count_as_execution (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_setup_calling_body_does_not_count_as_execution) ... ok +test_subject_helper_callback_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_subject_helper_callback_assertions_cannot_kill) ... ok +test_synchronous_returning_deferred_or_value_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_synchronous_returning_deferred_or_value_refused) ... ok +test_missing_and_malformed_source_write_failure_report (tests.test_current_site_browser_contract.BrowserReportFailureTests.test_missing_and_malformed_source_write_failure_report) ... ok +test_published_overlay_and_snapshot_only_static_reading_have_separate_exact_expectations (tests.test_current_site_browser_contract.ResumeContractTests.test_published_overlay_and_snapshot_only_static_reading_have_separate_exact_expectations) ... ok +test_wrong_shared_fields_phantom_links_and_false_operational_state_are_refused (tests.test_current_site_browser_contract.ResumeContractTests.test_wrong_shared_fields_phantom_links_and_false_operational_state_are_refused) ... ok +test_browser_dependencies_are_separate_and_version_pinned (tests.test_public_shop_browser_contract.BrowserCheckContract.test_browser_dependencies_are_separate_and_version_pinned) ... ok +test_browser_job_is_read_only_and_retains_failure_artifacts (tests.test_public_shop_browser_contract.BrowserCheckContract.test_browser_job_is_read_only_and_retains_failure_artifacts) ... ok +test_runner_is_loopback_only_with_a_four_case_matrix (tests.test_public_shop_browser_contract.BrowserCheckContract.test_runner_is_loopback_only_with_a_four_case_matrix) ... ok +test_bibtex_browser_checks_compare_real_clipboard_and_keep_failure_paths (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_bibtex_browser_checks_compare_real_clipboard_and_keep_failure_paths) ... ok +test_bibtex_disabled_checks_keep_the_hidden_button_addressable (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_bibtex_disabled_checks_keep_the_hidden_button_addressable) ... ok +test_clipboard_denial_fixture_reinitializes_after_same_document_history (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_clipboard_denial_fixture_reinitializes_after_same_document_history) ... ok +test_existing_runner_covers_source_quote_and_keyboard_disclosure (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_existing_runner_covers_source_quote_and_keyboard_disclosure) ... ok +test_new_reader_controls_are_reached_by_the_existing_browser_flows (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_new_reader_controls_are_reached_by_the_existing_browser_flows) ... ok +test_other_teaching_exports_are_reached_in_the_four_source_cases (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_other_teaching_exports_are_reached_in_the_four_source_cases) ... ok +test_pin_source_enumeration_preserves_the_downloaded_artifact_path (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_pin_source_enumeration_preserves_the_downloaded_artifact_path) ... ok +test_reader_tools_have_desktop_and_mobile_browser_evidence (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_reader_tools_have_desktop_and_mobile_browser_evidence) ... ok +test_rendered_layout_cases_read_what_the_browser_drew (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_rendered_layout_cases_read_what_the_browser_drew) ... ok + +---------------------------------------------------------------------- +Ran 45 tests in 0.690s + +OK diff --git a/docs/cbz/evidence/browser-origin-20261029/original-false-accept-normal.log b/docs/cbz/evidence/browser-origin-20261029/original-false-accept-normal.log new file mode 100644 index 000000000..da9318d13 --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/original-false-accept-normal.log @@ -0,0 +1,2 @@ +BROWSER_FALSE_ACCEPT: six setup-only failures accepted as mutant kills +DD_FALSE_ACCEPT: subject-origin AssertionError accepted as mutant kill diff --git a/docs/cbz/evidence/browser-origin-20261029/original-false-accept-optimized.log b/docs/cbz/evidence/browser-origin-20261029/original-false-accept-optimized.log new file mode 100644 index 000000000..da9318d13 --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/original-false-accept-optimized.log @@ -0,0 +1,2 @@ +BROWSER_FALSE_ACCEPT: six setup-only failures accepted as mutant kills +DD_FALSE_ACCEPT: subject-origin AssertionError accepted as mutant kill diff --git a/docs/cbz/evidence/browser-origin-20261029/preservation.json b/docs/cbz/evidence/browser-origin-20261029/preservation.json new file mode 100644 index 000000000..633520648 --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/preservation.json @@ -0,0 +1,15 @@ +{ + "base": "1390112fe58093d906c878043d6b985067610002", + "original_test_methods": 17, + "retained_method_names": true, + "changed_original_methods": [ + "test_six_runner_source_mutants_are_killed_by_assertions" + ], + "protected_paths_unchanged": [ + "tools/run_atlas_browser_regressions.py", + "tests/test_atlas_focus_geometry.py", + "tests/test_atlas_reader_scroll.py", + ".github/workflows/public-shop.yml" + ], + "test_source_sha256": "2d2b2d477f632cf86096d623cf6b29616c52e3cf89328b280f995acdeeefb0c6" +} diff --git a/tests/test_atlas_browser_regressions_runner.py b/tests/test_atlas_browser_regressions_runner.py index 7b9742145..be1e90367 100644 --- a/tests/test_atlas_browser_regressions_runner.py +++ b/tests/test_atlas_browser_regressions_runner.py @@ -2,6 +2,7 @@ import ast import importlib.util import io +import inspect import json import tempfile import types @@ -21,6 +22,107 @@ def load_runner(): return module + +# Finite trusted-unittest assertion helpers. File names and formatted tracebacks +# are never authority for an oracle failure; actual code identities are used. +_ASSERTION_CODES = frozenset( + value.__code__ for cls in (unittest.TestCase, unittest.case._AssertRaisesContext, unittest.case._AssertRaisesBaseContext, unittest.case._BaseTestCaseContext) + for name, value in vars(cls).items() + if callable(value) and hasattr(value, '__code__') + and (name.startswith('assert') or name in + ('fail', '_baseAssertEqual', '_formatMessage', '__exit__', '_raiseFailure')) +) + + +def run_oracle_controls(case_type, methods): + """Run frozen synchronous bodies and capture origin before unittest formats it.""" + expected = [case_type(method).id() for method in methods] + if not expected or len(set(expected)) != len(expected): + raise ValueError('nonempty unique oracle inventory required') + states = {} + class OriginResult(unittest.TextTestResult): + def __init__(self, *args): + super().__init__(*args) + self.started = [] + self.stopped = [] + self.origins = [] + self.successful = [] + def startTest(self, test): + self.started.append(test.id()) + super().startTest(test) + def stopTest(self, test): + self.stopped.append(test.id()) + super().stopTest(test) + def addSuccess(self, test): + self.successful.append(test.id()) + super().addSuccess(test) + def capture(self, test, err): + state = states.get(test.id(), {}) + codes = [] + tb = err[2] + while tb is not None: + codes.append(tb.tb_frame.f_code) + tb = tb.tb_next + body = state.get('code') + positions = [i for i, code in enumerate(codes) if code is body] + valid = (type(err[1]) is AssertionError and state.get('entered') == 1 + and state.get('phase') in ('body', 'body-exited') + and len(positions) == 1 + and all(code in _ASSERTION_CODES for code in codes[positions[0]+1:])) + self.origins.append((test.id(), valid)) + def addFailure(self, test, err): + self.capture(test, err) + super().addFailure(test, err) + def addSubTest(self, test, subtest, err): + if err is not None: + self.capture(test, err) + super().addSubTest(test, subtest, err) + cases = [] + for method in methods: + case = case_type(method) + selected = getattr(case, method) + code = getattr(selected, '__code__', None) + if code is None or inspect.iscoroutinefunction(selected) or inspect.isgeneratorfunction(selected) or inspect.isasyncgenfunction(selected): + raise ValueError('only synchronous oracle bodies are admitted') + state = dict(code=code, entered=0, phase='initial') + states[case.id()] = state + for attr, phase in (('_callSetUp', 'setup'), ('_callTearDown', 'teardown'), ('_callCleanup', 'cleanup')): + original = getattr(case, attr) + def call(*args, _original=original, _phase=phase, _state=state, **kwargs): + _state['phase'] = _phase + return _original(*args, **kwargs) + setattr(case, attr, call) + original = case._callTestMethod + def body_call(method, _original=original, _state=state): + if getattr(method, '__code__', None) is not _state['code']: + raise RuntimeError('oracle body changed') + _state['entered'] += 1 + _state['phase'] = 'body' + try: + returned = method() + if returned is not None: + if inspect.iscoroutine(returned) or inspect.isgenerator(returned): + returned.close() + raise RuntimeError('oracle body returned a deferred or unsupported value') + return None + finally: + _state['phase'] = 'body-exited' + case._callTestMethod = body_call + cases.append(case) + result = unittest.TextTestRunner(stream=io.StringIO(), resultclass=OriginResult).run(unittest.TestSuite(cases)) + complete = (result.testsRun == len(expected) and result.started == expected + and result.stopped == expected and all(s['entered'] == 1 for s in states.values()) + and not result.errors and not result.skipped and not result.expectedFailures + and not result.unexpectedSuccesses) + accounted = set(result.successful) | {identity for identity, unused in result.origins} + complete = (complete and accounted == set(expected) + and len(result.successful) == len(set(result.successful)) + and not (set(result.successful) & {identity for identity, unused in result.origins})) + result.oracle_baseline = complete and result.wasSuccessful() and not result.origins + result.oracle_killed = complete and bool(result.failures) and bool(result.origins) and all(v for _, v in result.origins) + return result + + class AtlasBrowserRunnerControls(unittest.TestCase): def setUp(self): self.runner = load_runner() @@ -149,9 +251,11 @@ def test_six_runner_source_mutants_are_killed_by_assertions(self): spec = importlib.util.spec_from_file_location('mutant_' + name, path) mutant = importlib.util.module_from_spec(spec) spec.loader.exec_module(mutant) + baseline = run_oracle_controls(type(self), (method,)) + self.assertTrue(baseline.oracle_baseline, name) with mock.patch(__name__ + '.load_runner', return_value=mutant): - result = unittest.TextTestRunner(stream=io.StringIO()).run( - unittest.TestSuite([type(self)(method)])) + result = run_oracle_controls(type(self), (method,)) + self.assertTrue(result.oracle_killed, name) self.assertGreater(len(result.failures), 0, name) self.assertEqual(result.errors, [], name) self.assertEqual(result.skipped, [], name) @@ -229,5 +333,134 @@ def test_workflow_runs_both_modes_and_retains_evidence(self): self.assertIn('if: always()', source) -if __name__ == '__main__': +class OracleOriginControls(unittest.TestCase): + def observe(self, body, **attributes): + cls = type('FrozenOracle', (unittest.TestCase,), dict(test_body=body, **attributes)) + return run_oracle_controls(cls, ('test_body',)) + + def test_genuine_baseline_and_assertion_origins(self): + def positive(case): case.assertEqual(1, 1) + def failure(case): case.assertEqual(1, 2) + def subtest(case): + with case.subTest(value=1): case.assertFalse(True) + def context(case): + with case.assertRaises(ValueError): pass + self.assertTrue(self.observe(positive).oracle_baseline) + for body in (failure, subtest, context): + with self.subTest(body=body.__name__): + result = self.observe(body) + self.assertTrue(result.oracle_killed) + self.assertFalse(result.oracle_baseline) + + def test_fixture_assertions_cannot_kill(self): + def positive(case): pass + def failure(case): case.fail('actual body') + def fixture(case): case.fail('fixture') + for attr in ('setUp', 'tearDown'): + for body in (positive, failure): + with self.subTest(attr=attr, body=body.__name__): + result = self.observe(body, **{attr: fixture}) + self.assertFalse(result.oracle_killed) + self.assertFalse(result.oracle_baseline) + + def test_later_phase_calling_selected_body_cannot_kill(self): + def body(case): case.fail('selected body') + def later(case): case.test_body() + self.assertFalse(self.observe(body, tearDown=later).oracle_killed) + def setup(case): case.addCleanup(case.test_body) + self.assertFalse(self.observe(body, setUp=setup).oracle_killed) + + def test_cleanup_assertions_cannot_kill(self): + def cleanup(): raise AssertionError('cleanup') + def body(case): + case.addCleanup(cleanup) + case.fail('body') + self.assertFalse(self.observe(body).oracle_killed) + + def test_subject_helper_callback_assertions_cannot_kill(self): + def subject(): raise AssertionError('subject') + def subject_body(case): subject() + def helper(case): case.fail('helper') + def helper_body(case): case.helper() + def callback_body(case): + callback = lambda: (_ for _ in ()).throw(AssertionError('callback')) + callback() + for body in (subject_body, helper_body, callback_body): + with self.subTest(body=body.__name__): + self.assertFalse(self.observe(body, helper=helper).oracle_killed) + + def test_forged_unittest_filename_is_not_authority(self): + namespace = {} + exec(compile("def forged(): raise AssertionError('forged')", unittest.case.__file__, 'exec'), namespace) + def body(case): namespace['forged']() + self.assertFalse(self.observe(body).oracle_killed) + + def test_setup_calling_body_does_not_count_as_execution(self): + def body(case): case.fail('called from setup') + def setup(case): case.test_body() + self.assertFalse(self.observe(body, setUp=setup).oracle_killed) + + def test_error_skip_expected_failure_refused(self): + def error(case): raise RuntimeError('error') + def skip(case): case.skipTest('skip') + @unittest.expectedFailure + def expected(case): case.fail('expected') + @unittest.expectedFailure + def unexpected(case): pass + for body in (error, skip, expected, unexpected): + with self.subTest(body=body.__name__): + result = self.observe(body) + self.assertFalse(result.oracle_killed) + self.assertFalse(result.oracle_baseline) + + def test_deferred_bodies_refused(self): + async def coroutine(case): pass + def generator(case): yield 1 + async def async_generator(case): yield 1 + for body in (coroutine, generator, async_generator): + with self.subTest(body=body.__name__), self.assertRaisesRegex(ValueError, 'synchronous'): + self.observe(body) + + def test_synchronous_returning_deferred_or_value_refused(self): + async def deferred(): pass + def coroutine(case): return deferred() + def generator(case): return iter((1,)) + def value(case): return 7 + for body in (coroutine, generator, value): + with self.subTest(body=body.__name__): + result = self.observe(body) + self.assertFalse(result.oracle_baseline) + self.assertFalse(result.oracle_killed) + + def test_empty_duplicate_population_refused(self): + cls = type('FrozenOracle', (unittest.TestCase,), {'test_body': lambda case: None}) + for methods in ((), ('test_body', 'test_body')): + with self.subTest(methods=methods), self.assertRaisesRegex(ValueError, 'inventory'): + run_oracle_controls(cls, methods) + + def test_repeated_body_execution_refused(self): + def body(case): case.fail('body') + def run(case, result): + result.startTest(case) + for unused in range(2): + try: case._callTestMethod(case.test_body) + except AssertionError: + import sys + result.addFailure(case, sys.exc_info()) + result.stopTest(case) + self.assertFalse(self.observe(body, run=run).oracle_killed) + + def test_missing_or_duplicate_terminal_success_refused(self): + def body(case): pass + for count in (0, 2): + def run(case, result): + result.startTest(case) + case._callTestMethod(case.test_body) + for unused in range(count): result.addSuccess(case) + result.stopTest(case) + with self.subTest(count=count): + self.assertFalse(self.observe(body, run=run).oracle_baseline) + + +if __name__ == "__main__": unittest.main() From 3154eaf54922ee512b584038d7dad28ebd843bd8 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:37:45 -0500 Subject: [PATCH 3/3] Reject structurally equal cloned code in exact oracle identity checks --- docs/cbz/ATLAS_BROWSER_REGRESSIONS.md | 16 ++++++ .../browser-code-identity-20261029/README.md | 16 ++++++ .../SHA256SUMS.json | 11 ++++ .../mutations-normal.json | 55 +++++++++++++++++++ .../mutations-normal.stderr | 3 + .../mutations-optimized.json | 55 +++++++++++++++++++ .../mutations-optimized.stderr | 3 + .../browser-code-identity-20261029/normal.log | 51 +++++++++++++++++ .../optimized.log | 51 +++++++++++++++++ .../preservation.json | 15 +++++ .../browser-code-identity-20261029/red.log | 14 +++++ .../check_grader_mutations.py | 3 +- .../test_atlas_browser_regressions_runner.py | 14 ++++- 13 files changed, 304 insertions(+), 3 deletions(-) create mode 100644 docs/cbz/evidence/browser-code-identity-20261029/README.md create mode 100644 docs/cbz/evidence/browser-code-identity-20261029/SHA256SUMS.json create mode 100644 docs/cbz/evidence/browser-code-identity-20261029/mutations-normal.json create mode 100644 docs/cbz/evidence/browser-code-identity-20261029/mutations-normal.stderr create mode 100644 docs/cbz/evidence/browser-code-identity-20261029/mutations-optimized.json create mode 100644 docs/cbz/evidence/browser-code-identity-20261029/mutations-optimized.stderr create mode 100644 docs/cbz/evidence/browser-code-identity-20261029/normal.log create mode 100644 docs/cbz/evidence/browser-code-identity-20261029/optimized.log create mode 100644 docs/cbz/evidence/browser-code-identity-20261029/preservation.json create mode 100644 docs/cbz/evidence/browser-code-identity-20261029/red.log diff --git a/docs/cbz/ATLAS_BROWSER_REGRESSIONS.md b/docs/cbz/ATLAS_BROWSER_REGRESSIONS.md index e7b019b57..33d6fc5eb 100644 --- a/docs/cbz/ATLAS_BROWSER_REGRESSIONS.md +++ b/docs/cbz/ATLAS_BROWSER_REGRESSIONS.md @@ -147,3 +147,19 @@ this is not hostile-process containment or an exhaustive proof of unittest. New composed-tree full regression/actual-browser execution remains pending. Canonical tooling registration should include this amendment and its self-tests, not relabel historical source receipts as current-source qualification. + + +## Exact code-object identity correction + +The subsequent reviewer found that Python CodeType objects can be structurally +`==` despite distinct identities, including after `co_filename` replacement. +At published parent `8d24c37ff52c012294d71bbcbf08bd24fae4e22a`, set membership +through a frozenset used equality and admitted such a cloned assertion helper. +The new clone regression first failed on that parent. Approved code references +are now retained in a tuple and checked with explicit `is`, never equality or +hash-set membership. All prior controls remain; current counts are31 module +methods /46 combined methods per mode, plus seven grader safeguard mutants. +Evidence is in `evidence/browser-code-identity-20261029/`. Earlier30/45-method +logs and six-mutant observations remain historical at their original source. +This closes the exact-provenance contract defect, not a mathematical proof gap; +no evidence shows earlier actual finite kills or browser outputs were false. diff --git a/docs/cbz/evidence/browser-code-identity-20261029/README.md b/docs/cbz/evidence/browser-code-identity-20261029/README.md new file mode 100644 index 000000000..1cff7a1d8 --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/README.md @@ -0,0 +1,16 @@ +# Exact code-object identity repair + +Parent: `8d24c37ff52c012294d71bbcbf08bd24fae4e22a`. A newly added regression first produced an actual assertion +failure: `fail.__code__.replace(co_filename='subject_fake.py')` is a distinct +but structurally equal code object and was accepted by equality-based membership. +The RED log is retained. Tuple storage plus explicit identity comparisons now +refuse the clone. A grader mutation restores equality and is caught by the new +regression, graded through the pristine structured origin classifier. + +New observed population: 31 module /46 combined controls per mode; seven grader safeguard mutations. +Commands remain in the parent tool verification documentation; these tests do +not launch a browser or perform a full certificate replay. Pinned CPython3.11.16 +is used for final local runs. Original production/workflow/packet bytes and all +prior test methods are preserved. Previous evidence is retained at its original +source, not relabeled as current. No full repository, hosted, mathematical or +CBZ-0 certification is claimed. diff --git a/docs/cbz/evidence/browser-code-identity-20261029/SHA256SUMS.json b/docs/cbz/evidence/browser-code-identity-20261029/SHA256SUMS.json new file mode 100644 index 000000000..3aca4a579 --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/SHA256SUMS.json @@ -0,0 +1,11 @@ +{ + "README.md": "98b2b00935b847e126944630a95f34fa0b2854afa2050553b45626995eeae1ba", + "mutations-normal.json": "57cb7a862d89111ef676a2e060cd2aad99256512c8165f1ae3740506eff550fa", + "mutations-normal.stderr": "1dcc6bb9d77f7e6f02fc2f854fa351c6a54ad682e8f46f7a65ebe9ed2b1d0fc0", + "mutations-optimized.json": "6120a43f901b7eeefffc87e5c5153516e71a6117c13b15b3b1c3ef71ec813d5a", + "mutations-optimized.stderr": "1dcc6bb9d77f7e6f02fc2f854fa351c6a54ad682e8f46f7a65ebe9ed2b1d0fc0", + "normal.log": "9c4e5cb39ae0b15c461e0dd4833928573e0fbf5425140ac2309f2b4c36df4907", + "optimized.log": "abcea7a39ca72e1b811dadaadd4f549c3d20710f2d9647d1a64ef3f3d030283e", + "preservation.json": "1762f2bc570851b196f5d7a73178a778c8bf16f06de6c07e42a6b1c6b65729ff", + "red.log": "e337ad79ba81713bdfe005c90f22354fd5119544ec63c9b6b764fa3ad4cafdab" +} diff --git a/docs/cbz/evidence/browser-code-identity-20261029/mutations-normal.json b/docs/cbz/evidence/browser-code-identity-20261029/mutations-normal.json new file mode 100644 index 000000000..b53df583a --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/mutations-normal.json @@ -0,0 +1,55 @@ +{ + "optimized": 0, + "source_sha256": "08ca3dbdaf40e7a48caf658c431d4242c3ec95cf0657bf318357ea17fbf2ec30", + "results": [ + { + "name": "code_identity", + "method": "test_cloned_approved_code_is_not_authorized_identity", + "intended_assertion_kill": true, + "failures": 1, + "errors": 0 + }, + { + "name": "origin_frames", + "method": "test_subject_helper_callback_assertions_cannot_kill", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "phase", + "method": "test_later_phase_calling_selected_body_cannot_kill", + "intended_assertion_kill": true, + "failures": 1, + "errors": 0 + }, + { + "name": "origin_admission", + "method": "test_subject_helper_callback_assertions_cannot_kill", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "lazy_bodies", + "method": "test_deferred_bodies_refused", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "baseline", + "method": "test_error_skip_expected_failure_refused", + "intended_assertion_kill": true, + "failures": 4, + "errors": 0 + }, + { + "name": "terminal_accounting", + "method": "test_missing_or_duplicate_terminal_success_refused", + "intended_assertion_kill": true, + "failures": 2, + "errors": 0 + } + ] +} diff --git a/docs/cbz/evidence/browser-code-identity-20261029/mutations-normal.stderr b/docs/cbz/evidence/browser-code-identity-20261029/mutations-normal.stderr new file mode 100644 index 000000000..d47f219ea --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/mutations-normal.stderr @@ -0,0 +1,3 @@ +/workspace/shared/cbz-recovery/dd/runtime/python/lib/python3.11/unittest/case.py:623: RuntimeWarning: coroutine 'OracleOriginControls.test_deferred_bodies_refused..coroutine' was never awaited + self._callTestMethod(testMethod) +RuntimeWarning: Enable tracemalloc to get the object allocation traceback diff --git a/docs/cbz/evidence/browser-code-identity-20261029/mutations-optimized.json b/docs/cbz/evidence/browser-code-identity-20261029/mutations-optimized.json new file mode 100644 index 000000000..b57572997 --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/mutations-optimized.json @@ -0,0 +1,55 @@ +{ + "optimized": 1, + "source_sha256": "08ca3dbdaf40e7a48caf658c431d4242c3ec95cf0657bf318357ea17fbf2ec30", + "results": [ + { + "name": "code_identity", + "method": "test_cloned_approved_code_is_not_authorized_identity", + "intended_assertion_kill": true, + "failures": 1, + "errors": 0 + }, + { + "name": "origin_frames", + "method": "test_subject_helper_callback_assertions_cannot_kill", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "phase", + "method": "test_later_phase_calling_selected_body_cannot_kill", + "intended_assertion_kill": true, + "failures": 1, + "errors": 0 + }, + { + "name": "origin_admission", + "method": "test_subject_helper_callback_assertions_cannot_kill", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "lazy_bodies", + "method": "test_deferred_bodies_refused", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "baseline", + "method": "test_error_skip_expected_failure_refused", + "intended_assertion_kill": true, + "failures": 4, + "errors": 0 + }, + { + "name": "terminal_accounting", + "method": "test_missing_or_duplicate_terminal_success_refused", + "intended_assertion_kill": true, + "failures": 2, + "errors": 0 + } + ] +} diff --git a/docs/cbz/evidence/browser-code-identity-20261029/mutations-optimized.stderr b/docs/cbz/evidence/browser-code-identity-20261029/mutations-optimized.stderr new file mode 100644 index 000000000..d47f219ea --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/mutations-optimized.stderr @@ -0,0 +1,3 @@ +/workspace/shared/cbz-recovery/dd/runtime/python/lib/python3.11/unittest/case.py:623: RuntimeWarning: coroutine 'OracleOriginControls.test_deferred_bodies_refused..coroutine' was never awaited + self._callTestMethod(testMethod) +RuntimeWarning: Enable tracemalloc to get the object allocation traceback diff --git a/docs/cbz/evidence/browser-code-identity-20261029/normal.log b/docs/cbz/evidence/browser-code-identity-20261029/normal.log new file mode 100644 index 000000000..e20a971ca --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/normal.log @@ -0,0 +1,51 @@ +test_both_tests_select_override_then_chrome_then_chromium (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_both_tests_select_override_then_chrome_then_chromium) ... ok +test_duplicate_inventory_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_duplicate_inventory_refused) ... ok +test_empty_inventory_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_empty_inventory_refused) ... ok +test_exact_two_successes_pass (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_exact_two_successes_pass) ... ok +test_expected_failure_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_expected_failure_is_not_success) ... ok +test_failures_errors_and_subtests_are_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_failures_errors_and_subtests_are_not_success) ... ok +test_focus_and_reader_explicitly_enable_sandbox (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_focus_and_reader_explicitly_enable_sandbox) ... ok +test_main_success_and_failure_exit_match_new_report (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_main_success_and_failure_exit_match_new_report) ... ok +test_missing_execution_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_missing_execution_is_not_success) ... ok +test_reused_output_is_refused_without_overwrite (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_reused_output_is_refused_without_overwrite) ... ok +test_runtime_identity_records_each_test_selected_launcher (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_runtime_identity_records_each_test_selected_launcher) ... ok +test_six_runner_source_mutants_are_killed_by_assertions (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_six_runner_source_mutants_are_killed_by_assertions) ... ok +test_skip_is_failure (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_skip_is_failure) ... ok +test_source_drift_cannot_leave_successful_report (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_source_drift_cannot_leave_successful_report) ... ok +test_unexpected_success_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_unexpected_success_is_not_success) ... ok +test_workflow_runs_both_modes_and_retains_evidence (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_workflow_runs_both_modes_and_retains_evidence) ... ok +test_wrong_and_extra_ids_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_wrong_and_extra_ids_refused) ... ok +test_cleanup_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_cleanup_assertions_cannot_kill) ... ok +test_cloned_approved_code_is_not_authorized_identity (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_cloned_approved_code_is_not_authorized_identity) ... ok +test_deferred_bodies_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_deferred_bodies_refused) ... ok +test_empty_duplicate_population_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_empty_duplicate_population_refused) ... ok +test_error_skip_expected_failure_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_error_skip_expected_failure_refused) ... ok +test_fixture_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_fixture_assertions_cannot_kill) ... ok +test_forged_unittest_filename_is_not_authority (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_forged_unittest_filename_is_not_authority) ... ok +test_genuine_baseline_and_assertion_origins (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_genuine_baseline_and_assertion_origins) ... ok +test_later_phase_calling_selected_body_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_later_phase_calling_selected_body_cannot_kill) ... ok +test_missing_or_duplicate_terminal_success_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_missing_or_duplicate_terminal_success_refused) ... ok +test_repeated_body_execution_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_repeated_body_execution_refused) ... ok +test_setup_calling_body_does_not_count_as_execution (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_setup_calling_body_does_not_count_as_execution) ... ok +test_subject_helper_callback_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_subject_helper_callback_assertions_cannot_kill) ... ok +test_synchronous_returning_deferred_or_value_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_synchronous_returning_deferred_or_value_refused) ... ok +test_missing_and_malformed_source_write_failure_report (tests.test_current_site_browser_contract.BrowserReportFailureTests.test_missing_and_malformed_source_write_failure_report) ... ok +test_published_overlay_and_snapshot_only_static_reading_have_separate_exact_expectations (tests.test_current_site_browser_contract.ResumeContractTests.test_published_overlay_and_snapshot_only_static_reading_have_separate_exact_expectations) ... ok +test_wrong_shared_fields_phantom_links_and_false_operational_state_are_refused (tests.test_current_site_browser_contract.ResumeContractTests.test_wrong_shared_fields_phantom_links_and_false_operational_state_are_refused) ... ok +test_browser_dependencies_are_separate_and_version_pinned (tests.test_public_shop_browser_contract.BrowserCheckContract.test_browser_dependencies_are_separate_and_version_pinned) ... ok +test_browser_job_is_read_only_and_retains_failure_artifacts (tests.test_public_shop_browser_contract.BrowserCheckContract.test_browser_job_is_read_only_and_retains_failure_artifacts) ... ok +test_runner_is_loopback_only_with_a_four_case_matrix (tests.test_public_shop_browser_contract.BrowserCheckContract.test_runner_is_loopback_only_with_a_four_case_matrix) ... ok +test_bibtex_browser_checks_compare_real_clipboard_and_keep_failure_paths (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_bibtex_browser_checks_compare_real_clipboard_and_keep_failure_paths) ... ok +test_bibtex_disabled_checks_keep_the_hidden_button_addressable (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_bibtex_disabled_checks_keep_the_hidden_button_addressable) ... ok +test_clipboard_denial_fixture_reinitializes_after_same_document_history (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_clipboard_denial_fixture_reinitializes_after_same_document_history) ... ok +test_existing_runner_covers_source_quote_and_keyboard_disclosure (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_existing_runner_covers_source_quote_and_keyboard_disclosure) ... ok +test_new_reader_controls_are_reached_by_the_existing_browser_flows (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_new_reader_controls_are_reached_by_the_existing_browser_flows) ... ok +test_other_teaching_exports_are_reached_in_the_four_source_cases (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_other_teaching_exports_are_reached_in_the_four_source_cases) ... ok +test_pin_source_enumeration_preserves_the_downloaded_artifact_path (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_pin_source_enumeration_preserves_the_downloaded_artifact_path) ... ok +test_reader_tools_have_desktop_and_mobile_browser_evidence (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_reader_tools_have_desktop_and_mobile_browser_evidence) ... ok +test_rendered_layout_cases_read_what_the_browser_drew (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_rendered_layout_cases_read_what_the_browser_drew) ... ok + +---------------------------------------------------------------------- +Ran 46 tests in 0.910s + +OK diff --git a/docs/cbz/evidence/browser-code-identity-20261029/optimized.log b/docs/cbz/evidence/browser-code-identity-20261029/optimized.log new file mode 100644 index 000000000..883331784 --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/optimized.log @@ -0,0 +1,51 @@ +test_both_tests_select_override_then_chrome_then_chromium (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_both_tests_select_override_then_chrome_then_chromium) ... ok +test_duplicate_inventory_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_duplicate_inventory_refused) ... ok +test_empty_inventory_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_empty_inventory_refused) ... ok +test_exact_two_successes_pass (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_exact_two_successes_pass) ... ok +test_expected_failure_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_expected_failure_is_not_success) ... ok +test_failures_errors_and_subtests_are_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_failures_errors_and_subtests_are_not_success) ... ok +test_focus_and_reader_explicitly_enable_sandbox (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_focus_and_reader_explicitly_enable_sandbox) ... ok +test_main_success_and_failure_exit_match_new_report (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_main_success_and_failure_exit_match_new_report) ... ok +test_missing_execution_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_missing_execution_is_not_success) ... ok +test_reused_output_is_refused_without_overwrite (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_reused_output_is_refused_without_overwrite) ... ok +test_runtime_identity_records_each_test_selected_launcher (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_runtime_identity_records_each_test_selected_launcher) ... ok +test_six_runner_source_mutants_are_killed_by_assertions (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_six_runner_source_mutants_are_killed_by_assertions) ... ok +test_skip_is_failure (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_skip_is_failure) ... ok +test_source_drift_cannot_leave_successful_report (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_source_drift_cannot_leave_successful_report) ... ok +test_unexpected_success_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_unexpected_success_is_not_success) ... ok +test_workflow_runs_both_modes_and_retains_evidence (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_workflow_runs_both_modes_and_retains_evidence) ... ok +test_wrong_and_extra_ids_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_wrong_and_extra_ids_refused) ... ok +test_cleanup_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_cleanup_assertions_cannot_kill) ... ok +test_cloned_approved_code_is_not_authorized_identity (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_cloned_approved_code_is_not_authorized_identity) ... ok +test_deferred_bodies_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_deferred_bodies_refused) ... ok +test_empty_duplicate_population_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_empty_duplicate_population_refused) ... ok +test_error_skip_expected_failure_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_error_skip_expected_failure_refused) ... ok +test_fixture_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_fixture_assertions_cannot_kill) ... ok +test_forged_unittest_filename_is_not_authority (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_forged_unittest_filename_is_not_authority) ... ok +test_genuine_baseline_and_assertion_origins (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_genuine_baseline_and_assertion_origins) ... ok +test_later_phase_calling_selected_body_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_later_phase_calling_selected_body_cannot_kill) ... ok +test_missing_or_duplicate_terminal_success_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_missing_or_duplicate_terminal_success_refused) ... ok +test_repeated_body_execution_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_repeated_body_execution_refused) ... ok +test_setup_calling_body_does_not_count_as_execution (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_setup_calling_body_does_not_count_as_execution) ... ok +test_subject_helper_callback_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_subject_helper_callback_assertions_cannot_kill) ... ok +test_synchronous_returning_deferred_or_value_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_synchronous_returning_deferred_or_value_refused) ... ok +test_missing_and_malformed_source_write_failure_report (tests.test_current_site_browser_contract.BrowserReportFailureTests.test_missing_and_malformed_source_write_failure_report) ... ok +test_published_overlay_and_snapshot_only_static_reading_have_separate_exact_expectations (tests.test_current_site_browser_contract.ResumeContractTests.test_published_overlay_and_snapshot_only_static_reading_have_separate_exact_expectations) ... ok +test_wrong_shared_fields_phantom_links_and_false_operational_state_are_refused (tests.test_current_site_browser_contract.ResumeContractTests.test_wrong_shared_fields_phantom_links_and_false_operational_state_are_refused) ... ok +test_browser_dependencies_are_separate_and_version_pinned (tests.test_public_shop_browser_contract.BrowserCheckContract.test_browser_dependencies_are_separate_and_version_pinned) ... ok +test_browser_job_is_read_only_and_retains_failure_artifacts (tests.test_public_shop_browser_contract.BrowserCheckContract.test_browser_job_is_read_only_and_retains_failure_artifacts) ... ok +test_runner_is_loopback_only_with_a_four_case_matrix (tests.test_public_shop_browser_contract.BrowserCheckContract.test_runner_is_loopback_only_with_a_four_case_matrix) ... ok +test_bibtex_browser_checks_compare_real_clipboard_and_keep_failure_paths (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_bibtex_browser_checks_compare_real_clipboard_and_keep_failure_paths) ... ok +test_bibtex_disabled_checks_keep_the_hidden_button_addressable (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_bibtex_disabled_checks_keep_the_hidden_button_addressable) ... ok +test_clipboard_denial_fixture_reinitializes_after_same_document_history (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_clipboard_denial_fixture_reinitializes_after_same_document_history) ... ok +test_existing_runner_covers_source_quote_and_keyboard_disclosure (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_existing_runner_covers_source_quote_and_keyboard_disclosure) ... ok +test_new_reader_controls_are_reached_by_the_existing_browser_flows (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_new_reader_controls_are_reached_by_the_existing_browser_flows) ... ok +test_other_teaching_exports_are_reached_in_the_four_source_cases (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_other_teaching_exports_are_reached_in_the_four_source_cases) ... ok +test_pin_source_enumeration_preserves_the_downloaded_artifact_path (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_pin_source_enumeration_preserves_the_downloaded_artifact_path) ... ok +test_reader_tools_have_desktop_and_mobile_browser_evidence (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_reader_tools_have_desktop_and_mobile_browser_evidence) ... ok +test_rendered_layout_cases_read_what_the_browser_drew (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_rendered_layout_cases_read_what_the_browser_drew) ... ok + +---------------------------------------------------------------------- +Ran 46 tests in 1.068s + +OK diff --git a/docs/cbz/evidence/browser-code-identity-20261029/preservation.json b/docs/cbz/evidence/browser-code-identity-20261029/preservation.json new file mode 100644 index 000000000..66725f572 --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/preservation.json @@ -0,0 +1,15 @@ +{ + "parent": "8d24c37ff52c012294d71bbcbf08bd24fae4e22a", + "source_sha256": "08ca3dbdaf40e7a48caf658c431d4242c3ec95cf0657bf318357ea17fbf2ec30", + "all_original_named_tests_retained": true, + "modified_original_methods": [], + "protected_paths_unchanged": [ + "tools/run_atlas_browser_regressions.py", + "tests/test_atlas_focus_geometry.py", + "tests/test_atlas_reader_scroll.py", + ".github/workflows/public-shop.yml" + ], + "new_test_methods": [ + "OracleOriginControls.test_cloned_approved_code_is_not_authorized_identity" + ] +} diff --git a/docs/cbz/evidence/browser-code-identity-20261029/red.log b/docs/cbz/evidence/browser-code-identity-20261029/red.log new file mode 100644 index 000000000..2c9334c1f --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/red.log @@ -0,0 +1,14 @@ +test_cloned_approved_code_is_not_authorized_identity (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_cloned_approved_code_is_not_authorized_identity) ... FAIL + +====================================================================== +FAIL: test_cloned_approved_code_is_not_authorized_identity (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_cloned_approved_code_is_not_authorized_identity) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/workspace/shared/cbz-recovery/oracle-audit/browser/tests/test_atlas_browser_regressions_runner.py", line 400, in test_cloned_approved_code_is_not_authorized_identity + self.assertFalse(self.observe(body).oracle_killed) +AssertionError: True is not false + +---------------------------------------------------------------------- +Ran 1 test in 0.001s + +FAILED (failures=1) diff --git a/docs/cbz/evidence/browser-origin-20261029/check_grader_mutations.py b/docs/cbz/evidence/browser-origin-20261029/check_grader_mutations.py index 5fcfe88c7..e48c6ba70 100644 --- a/docs/cbz/evidence/browser-origin-20261029/check_grader_mutations.py +++ b/docs/cbz/evidence/browser-origin-20261029/check_grader_mutations.py @@ -6,7 +6,8 @@ def load(name,path): spec=importlib.util.spec_from_file_location(name,path);m=importlib.util.module_from_spec(spec);sys.modules[name]=m;spec.loader.exec_module(m);return m trusted=load('trusted_browser_origin',PATH);source=PATH.read_text();rows=[] cases=[ - ('origin_frames','and all(code in _ASSERTION_CODES for code in codes[positions[0]+1:])','and True','test_subject_helper_callback_assertions_cannot_kill'), + ('code_identity','any(code is allowed for allowed in _ASSERTION_CODES)','code in _ASSERTION_CODES','test_cloned_approved_code_is_not_authorized_identity'), + ('origin_frames','and all(any(code is allowed for allowed in _ASSERTION_CODES) for code in codes[positions[0]+1:])','and True','test_subject_helper_callback_assertions_cannot_kill'), ('phase',"and state.get('phase') in ('body', 'body-exited')",'and True','test_later_phase_calling_selected_body_cannot_kill'), ('origin_admission','and all(v for _, v in result.origins)','and True','test_subject_helper_callback_assertions_cannot_kill'), ('lazy_bodies',' or inspect.iscoroutinefunction(selected) or inspect.isgeneratorfunction(selected) or inspect.isasyncgenfunction(selected)','', 'test_deferred_bodies_refused'), diff --git a/tests/test_atlas_browser_regressions_runner.py b/tests/test_atlas_browser_regressions_runner.py index be1e90367..d27833408 100644 --- a/tests/test_atlas_browser_regressions_runner.py +++ b/tests/test_atlas_browser_regressions_runner.py @@ -25,7 +25,7 @@ def load_runner(): # Finite trusted-unittest assertion helpers. File names and formatted tracebacks # are never authority for an oracle failure; actual code identities are used. -_ASSERTION_CODES = frozenset( +_ASSERTION_CODES = tuple( value.__code__ for cls in (unittest.TestCase, unittest.case._AssertRaisesContext, unittest.case._AssertRaisesBaseContext, unittest.case._BaseTestCaseContext) for name, value in vars(cls).items() if callable(value) and hasattr(value, '__code__') @@ -68,7 +68,7 @@ def capture(self, test, err): valid = (type(err[1]) is AssertionError and state.get('entered') == 1 and state.get('phase') in ('body', 'body-exited') and len(positions) == 1 - and all(code in _ASSERTION_CODES for code in codes[positions[0]+1:])) + and all(any(code is allowed for allowed in _ASSERTION_CODES) for code in codes[positions[0]+1:])) self.origins.append((test.id(), valid)) def addFailure(self, test, err): self.capture(test, err) @@ -389,6 +389,16 @@ def callback_body(case): with self.subTest(body=body.__name__): self.assertFalse(self.observe(body, helper=helper).oracle_killed) + def test_cloned_approved_code_is_not_authorized_identity(self): + original = unittest.TestCase.fail + cloned_code = original.__code__.replace(co_filename='subject_fake.py') + self.assertIsNot(cloned_code, original.__code__) + self.assertEqual(cloned_code, original.__code__) + cloned = types.FunctionType(cloned_code, original.__globals__, + original.__name__, original.__defaults__) + def body(case): cloned(case, 'cloned subject code') + self.assertFalse(self.observe(body).oracle_killed) + def test_forged_unittest_filename_is_not_authority(self): namespace = {} exec(compile("def forged(): raise AssertionError('forged')", unittest.case.__file__, 'exec'), namespace)