From 8189ca90de6e6785d5317360c141395231295b16 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:47:14 -0600 Subject: [PATCH] test(g11): exercise Atlas accessible-list browser check --- .github/workflows/public-shop.yml | 9 +- tests/test_atlas_accessible_list_mutations.py | 57 ++++++++ tests/test_atlas_accessible_list_runner.py | 133 ++++++++++++++++++ ...current_atlas_accessible_list_reference.py | 46 ++++++ tools/atlas_accessible_list_browser_check.py | 12 +- 5 files changed, 252 insertions(+), 5 deletions(-) create mode 100644 tests/test_atlas_accessible_list_mutations.py create mode 100644 tests/test_atlas_accessible_list_runner.py create mode 100644 tests/test_g11_current_atlas_accessible_list_reference.py diff --git a/.github/workflows/public-shop.yml b/.github/workflows/public-shop.yml index 22384eddc..35b5fbddc 100644 --- a/.github/workflows/public-shop.yml +++ b/.github/workflows/public-shop.yml @@ -78,10 +78,15 @@ jobs: python -B tools/atlas_accessibility_regression.py --site docs/site --output browser-evidence/atlas-accessibility-regression + - name: Atlas accessible-list metadata contrast + run: >- + python -B tools/atlas_accessible_list_browser_check.py + --site docs/site + --output browser-evidence/atlas-accessible-list - name: Exact Atlas browser runner controls run: | - python -B -S -m unittest tests.test_atlas_browser_regressions_runner tests.test_atlas_accessibility_regression_runner tests.test_atlas_accessibility_regression_mutations tests.test_g11_current_atlas_accessibility_regression_reference -v - python -B -O -S -m unittest tests.test_atlas_browser_regressions_runner tests.test_atlas_accessibility_regression_runner tests.test_atlas_accessibility_regression_mutations tests.test_g11_current_atlas_accessibility_regression_reference -v + python -B -S -m unittest tests.test_atlas_browser_regressions_runner tests.test_atlas_accessibility_regression_runner tests.test_atlas_accessibility_regression_mutations tests.test_g11_current_atlas_accessibility_regression_reference tests.test_atlas_accessible_list_runner tests.test_atlas_accessible_list_mutations tests.test_g11_current_atlas_accessible_list_reference -v + python -B -O -S -m unittest tests.test_atlas_browser_regressions_runner tests.test_atlas_accessibility_regression_runner tests.test_atlas_accessibility_regression_mutations tests.test_g11_current_atlas_accessibility_regression_reference tests.test_atlas_accessible_list_runner tests.test_atlas_accessible_list_mutations tests.test_g11_current_atlas_accessible_list_reference -v - name: Atlas focus and reader regressions in both modes run: | set -euo pipefail diff --git a/tests/test_atlas_accessible_list_mutations.py b/tests/test_atlas_accessible_list_mutations.py new file mode 100644 index 000000000..16f200981 --- /dev/null +++ b/tests/test_atlas_accessible_list_mutations.py @@ -0,0 +1,57 @@ +"""Adverse controls for the accessible-list browser launch policy.""" + +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +RUNNER = ROOT / "tools" / "atlas_accessible_list_browser_check.py" +TARGET = "tests.test_atlas_accessible_list_runner" + + +class AtlasAccessibleListMutationTests(unittest.TestCase): + def assert_mutation_killed(self, before, after, expected_test): + source = RUNNER.read_text() + self.assertEqual(source.count(before), 1) + mutated = source.replace(before, after) + with tempfile.TemporaryDirectory() as raw: + mutant = Path(raw) / RUNNER.name + mutant.write_text(mutated) + env = dict(os.environ) + env["ATLAS_ACCESSIBLE_LIST_RUNNER"] = str(mutant) + command = [sys.executable, "-B"] + if sys.flags.optimize: + command.append("-O") + command.extend(["-S", "-m", "unittest", TARGET, "-v"]) + result = subprocess.run( + command, + cwd=ROOT, + env=env, + capture_output=True, + text=True, + check=False, + ) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn(expected_test, result.stdout + result.stderr) + + def test_wrong_default_browser_channel_is_killed(self): + self.assert_mutation_killed( + "launch['channel'] = 'chrome'", + "launch['channel'] = 'chromium'", + "test_default_browser_uses_sandbox_capable_chrome_channel", + ) + + def test_disabled_chromium_sandbox_is_killed(self): + self.assert_mutation_killed( + "launch = {'chromium_sandbox': True,", + "launch = {'chromium_sandbox': False,", + "test_default_browser_uses_sandbox_capable_chrome_channel", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_atlas_accessible_list_runner.py b/tests/test_atlas_accessible_list_runner.py new file mode 100644 index 000000000..0e634370c --- /dev/null +++ b/tests/test_atlas_accessible_list_runner.py @@ -0,0 +1,133 @@ +"""Process-level controls for the Atlas accessible-list browser check.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import textwrap +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +RUNNER = Path( + os.environ.get( + "ATLAS_ACCESSIBLE_LIST_RUNNER", + ROOT / "tools" / "atlas_accessible_list_browser_check.py", + ) +) + + +class AtlasAccessibleListRunnerTests(unittest.TestCase): + def run_until_browser_launch(self, *, explicit_browser=False): + with tempfile.TemporaryDirectory() as raw: + work = Path(raw) + package = work / "python" / "playwright" + package.mkdir(parents=True) + browser = work / "browser" + browser.write_text("#!/bin/sh\nexit 0\n") + browser.chmod(0o755) + capture = work / "launch.json" + (package / "__init__.py").write_text("") + (package / "sync_api.py").write_text( + textwrap.dedent( + f""" + from pathlib import Path + + def expect(*_args, **_kwargs): + raise AssertionError("expect should not run before launch") + + class Chromium: + def launch(self, **kwargs): + import json + Path({str(capture)!r}).write_text(json.dumps(kwargs, sort_keys=True)) + raise SystemExit(73) + + class Playwright: + chromium = Chromium() + + class Context: + def __enter__(self): + return Playwright() + def __exit__(self, *_args): + return False + + def sync_playwright(): + return Context() + """ + ) + ) + output = work / "output" + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + [str(work / "python"), str(ROOT / "tools")] + ) + command = [sys.executable] + if sys.flags.optimize: + command.append("-O") + command.extend( + [ + str(RUNNER), + "--site", + str(ROOT / "docs" / "site"), + "--output", + str(output), + ] + ) + if explicit_browser: + command.extend(["--browser", str(browser)]) + result = subprocess.run( + command, + cwd=ROOT, + env=env, + capture_output=True, + text=True, + check=False, + ) + launch = json.loads(capture.read_text()) if capture.exists() else None + report_path = output / "contrast-browser.json" + report = json.loads(report_path.read_text()) if report_path.exists() else None + return result, launch, browser, report + + def test_default_browser_uses_sandbox_capable_chrome_channel(self): + result, launch, _browser, _report = self.run_until_browser_launch() + self.assertEqual(result.returncode, 73, result.stderr) + self.assertEqual( + launch, + { + "args": ["--enable-unsafe-swiftshader"], + "channel": "chrome", + "chromium_sandbox": True, + }, + ) + + def test_explicit_browser_uses_requested_executable(self): + result, launch, browser, _report = self.run_until_browser_launch( + explicit_browser=True + ) + self.assertEqual(result.returncode, 73, result.stderr) + self.assertEqual( + launch, + { + "args": ["--enable-unsafe-swiftshader"], + "chromium_sandbox": True, + "executable_path": str(browser), + }, + ) + + def test_report_records_default_launch_boundary_before_browser_work(self): + result, _launch, _browser, report = self.run_until_browser_launch() + self.assertEqual(result.returncode, 73, result.stderr) + self.assertEqual( + report["browser_launch"], + { + "args": ["--enable-unsafe-swiftshader"], + "channel": "chrome", + "chromium_sandbox": True, + }, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_g11_current_atlas_accessible_list_reference.py b/tests/test_g11_current_atlas_accessible_list_reference.py new file mode 100644 index 000000000..1a231e3a4 --- /dev/null +++ b/tests/test_g11_current_atlas_accessible_list_reference.py @@ -0,0 +1,46 @@ +"""Repository integration control for the accessible-list browser CLI.""" + +import sys +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools" / "cbz")) +import g11_census as g11 # noqa: E402 + + +class CurrentAtlasAccessibleListReference(unittest.TestCase): + paths = ( + "tools/atlas_accessible_list_browser_check.py", + ".github/workflows/public-shop.yml", + ) + + def census(self, replacements=None): + replacements = replacements or {} + files = { + path: replacements.get(path, (ROOT / path).read_bytes()) + for path in self.paths + } + return g11.census(files, sorted(files))["tools"] + + def test_accessible_list_browser_check_has_static_execution_reference(self): + report = self.census() + self.assertTrue( + report["tools/atlas_accessible_list_browser_check.py"]["referenced"] + ) + + def test_removing_workflow_invocation_is_detected(self): + workflow = (ROOT / self.paths[1]).read_bytes() + command = b"python -B tools/atlas_accessible_list_browser_check.py" + self.assertEqual(workflow.count(command), 1) + report = self.census( + {self.paths[1]: workflow.replace(command, b"python -B")} + ) + self.assertFalse( + report["tools/atlas_accessible_list_browser_check.py"]["referenced"] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/atlas_accessible_list_browser_check.py b/tools/atlas_accessible_list_browser_check.py index 83735a6d4..a2bc48747 100644 --- a/tools/atlas_accessible_list_browser_check.py +++ b/tools/atlas_accessible_list_browser_check.py @@ -25,7 +25,7 @@ def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('--site', type=Path, required=True) p.add_argument('--output', type=Path, required=True) - p.add_argument('--browser', default='/usr/bin/chromium') + p.add_argument('--browser') a = p.parse_args() a.output.mkdir(parents=True, exist_ok=True) class Handler(SimpleHTTPRequestHandler): @@ -39,8 +39,14 @@ def log_message(self, *unused): 'sandbox': True, 'renderer': 'Chromium SwiftShader software WebGL', 'cases': []} try: with sync_playwright() as pw: - browser = pw.chromium.launch(executable_path=a.browser, chromium_sandbox=True, - args=['--enable-unsafe-swiftshader']) + launch = {'chromium_sandbox': True, + 'args': ['--enable-unsafe-swiftshader']} + if a.browser: + launch['executable_path'] = a.browser + else: + launch['channel'] = 'chrome' + report['browser_launch'] = launch + browser = pw.chromium.launch(**launch) report['browser'] = browser.version for scheme in ['light', 'dark']: row = {'scheme': scheme, 'passed': False}