diff --git a/.github/workflows/public-shop.yml b/.github/workflows/public-shop.yml index 4bb75f288..e75bc07d7 100644 --- a/.github/workflows/public-shop.yml +++ b/.github/workflows/public-shop.yml @@ -68,6 +68,20 @@ jobs: run: python -B tools/public_shop_browser_check.py --output browser-evidence - name: Current mathematical atlas and static reading parity run: python -B tools/current_site_browser_check.py --output browser-evidence/current-atlas + - name: Exact Atlas browser runner controls + run: | + python -B -S -m unittest tests.test_atlas_browser_regressions_runner -v + python -B -O -S -m unittest tests.test_atlas_browser_regressions_runner -v + - name: Atlas focus and reader regressions in both modes + run: | + set -euo pipefail + mkdir -p browser-evidence/atlas-regressions + status=0 + python -B tools/run_atlas_browser_regressions.py --output browser-evidence/atlas-regressions/normal > browser-evidence/atlas-regressions/normal.stdout 2> browser-evidence/atlas-regressions/normal.stderr || status=1 + python -B -O tools/run_atlas_browser_regressions.py --output browser-evidence/atlas-regressions/optimized > browser-evidence/atlas-regressions/optimized.stdout 2> browser-evidence/atlas-regressions/optimized.stderr || status=1 + cat browser-evidence/atlas-regressions/*.stdout + cat browser-evidence/atlas-regressions/*.stderr >&2 + exit "$status" - name: Preserve original current-atlas report in bounded log frames if: always() timeout-minutes: 1 diff --git a/docs/cbz/ATLAS_BROWSER_REGRESSIONS.md b/docs/cbz/ATLAS_BROWSER_REGRESSIONS.md new file mode 100644 index 000000000..33d6fc5eb --- /dev/null +++ b/docs/cbz/ATLAS_BROWSER_REGRESSIONS.md @@ -0,0 +1,165 @@ +# Exact hosted Atlas browser regressions + +This engineering runner closes an execution-coverage gap for two existing UI +regressions. It does not certify mathematics, all regression coverage, or CBZ-0. +Actual browser execution passed on the corrected historical source documented below. +A later composition requires its own hosted qualification; historical success is +not transferred to an untested tree. + +## Scope and execution + +`tools/run_atlas_browser_regressions.py` loads only the exact focus-ring and +long-reader-selection method IDs declared in `EXPECTED_IDS`. It refuses an empty, +duplicate, missing, extra or renamed inventory. Success requires exactly two +successful methods, no failures/errors/skips, no expected failures/unexpected +successes, and unchanged clean tracked source identity before/after execution. +Both tests explicitly enable Chromium's sandbox and choose the explicit +`ATLAS_BROWSER_EXECUTABLE` override first, then packaged `google-chrome`, then +`chromium`. This matches the existing reader selection and hosted browser policy. +No UI assertion or bound is changed. The first hosted PR #702 attempt at source +`625bfc89975efa1fd725076bc2b0961af2768279` failed: focus selected Chromium and +reported “No usable sandbox”; reader passed with packaged Chrome. That failure +is not erased by this launcher correction. Corrected actual browser execution passed in both modes on the source below; +no local denied launch was retried. + +The existing public-shop `browser-smoke` job supplies the hash-pinned +`tests/browser-requirements.txt` dependencies and packaged Chrome. No permission, +action pin, dependency pin, browser flag disabling security, or browser download +is introduced. Existing browser checks remain before the new runner steps. + +After installing the existing pinned requirements in that supported environment: + +``` +python -B tools/run_atlas_browser_regressions.py --output browser-evidence/atlas-regressions/normal +python -B -O tools/run_atlas_browser_regressions.py --output browser-evidence/atlas-regressions/optimized +``` + +Each output directory must be new; reuse fails without overwriting old evidence. +Each mode records a result JSON and the test-produced screenshots/measurements. +CI separately retains complete stdout/stderr, runs both modes even if one fails, +and returns failure if either fails. The existing always-upload browser artifact +preserves these files. Runtime metadata records Python/Playwright and each chosen +browser launcher path, version output and launcher-file SHA256 (a launcher may be +a wrapper, so this is not a claim to hash every underlying browser binary). +Git commit/tree and tracked cleanliness are local observations, not cryptographic +attestation or hostile-process containment. A missing dependency, denied browser +launch, assertion failure or source drift is failure; do not relax the sandbox or +reinterpret an optional skip as success. The prior local socket denial is not +retried by this contribution. + +## Self-verification + +``` +python -B -S -m unittest tests.test_atlas_browser_regressions_runner -v +python -B -O -S -m unittest tests.test_atlas_browser_regressions_runner -v +``` + +Thirty controls run without launching a browser. The original seventeen are retained; thirteen additional controls qualify the mutation grader’s lifecycle and assertion-origin checks. They cover inventory, actual +unittest success/failure/error/subtest/skip/expected-failure/unexpected-success, +missing execution, report/exit consistency, source drift, old-output retention, +explicit sandbox parameters, workflow wiring, ten launcher-selection cases and +runner identity correspondence to each selected test launcher. Identity controls +stub version probing and never launch a browser. Six actual runner-source mutants +must fail their designated assertions with zero errors or skips: bypass inventory, +force green, ignore source drift, overwrite old evidence, return wrong exit and +retain a success report after exception. The unchanged control must pass. Finite +controls establish these contracts, not universal test-runner soundness. + +## Canonical registration addendum for the integrator + +Shared canonical files remain owned by the current baseline writer. Compose these +facts into their current versions rather than restoring the historical STATUS: + +- GOVERNANCE/TOOLING: `tools/run_atlas_browser_regressions.py`; purpose: fixed-ID, + zero-skip execution of two Atlas UI regressions; criterion: G5 execution coverage + and G11 documented tooling; usage/self-verification: exact commands above; + maintenance owner: none, self-contained; historical verification: 17 runner controls (32 combined browser-contract controls) and 6 + runner-source mutants per mode locally passed in the fresh recovery checkout; actual hosted execution passed on the historical corrected tree below, + with new composition qualification still pending. +- STATUS, How to verify: run both control commands, then both real runner commands + in the existing pinned browser-smoke environment; report the original result + JSON/logs and checked source identity. This does not close five separate + production-record applicability/live-source tests. +- DECISIONS: add the narrow execution-coverage runner because plain unittest can + exit zero after skipping required browser methods. Preserve all UI assertions, + source history, original failure logs and supported browser sandbox. No new UI + functionality, mathematical claim or final baseline promotion is introduced. + +## Retained hosted execution evidence + +The original failed run 40936994120 and corrected run 40950918974 are retained +in [the evidence manifest](evidence/atlas-browser-20261029/manifest.json). +Both downloaded artifact ZIPs were independently hashed against their recorded +SHA-256 identities before extracting these text records. Screenshot member hashes +are retained in the manifest; screenshots themselves remain in the hosted ZIPs. + +Corrected source `c0205ca9e8683b2847008f712582e183f723e186` was tested through +merge `8bd88e5952e4a67ff394e1cab3c272e7ace2f961`, tree +`c9e3a5656fc95feef3bd360af3ea393846afd1e8`, in browser-smoke job +123718493908. Its normal and optimized result records each contain exactly two +successful expected method IDs, zero errors, failures or skips, and optimize flags +0 and 1 respectively. Both used `/opt/google/chrome/google-chrome`, version +156.0.8078.47, with the existing sandbox-enabled tests. + +Corrected artifact 12931733641 has SHA-256 +`2bcd2bd3ceec0641c56f52bab6159e141e825a10042d9276bd716e7f12c1df36`. +Original artifact 12924209403 has SHA-256 +`e6ac7e329f63c55dc1188f6dddeacfb3a72b1f60a2327dcaac657cdaee0b3f8a`. +The original focus failure remains in its result and stderr records. These are +engineering observations only, not mathematical evidence or final G5 completion. +The later composition onto Main `1e2be44c69e08e47472041ac329e1b27f7dec793` +preserves this evidence as historical; it still needs a new hosted execution. + + +## 2026-10-29 assertion-origin amendment + +At parent tree `6786ec405cef50e0fef73c7fcd9ca3945c555230`, the six-mutant +meta-test accepted setup-only AssertionErrors as kills. An independent normal and +optimized reproduction is retained in `evidence/browser-origin-20261029/`. +This demonstrates a future-use grading defect; it does not establish that any +historical finite kill or actual browser result was false. + +The test-only grader now captures actual exception/code identity before unittest +formats/clears traceback data. Admission requires the selected synchronous body +exactly once, the complete terminal population, no fixture/error/skip/expected +failure, and only the selected body plus explicitly trusted unittest assertion +helper code on the escaping failure path. Subject, helper, callback, forged-file, +setup, teardown and cleanup assertions are refused. Deferred/non-None body returns +and missing/duplicate terminal success events are refused. Each production-source +mutant first runs its unmodified designated baseline. All original assertions, +production runner/browser sources, workflow bytes and sandbox policy are unchanged. + +Current scoped controls: 30 methods per mode, or 45 when combined with the two +existing browser-contract modules; six original production mutants per mode. +Six additional grader safeguard mutations are checked by an unchanged trusted +copy of the repaired raw-origin classifier, not by their own mutated classifier: + +``` +python -B docs/cbz/evidence/browser-origin-20261029/check_grader_mutations.py +python -B -O docs/cbz/evidence/browser-origin-20261029/check_grader_mutations.py +``` + +The deferred-body mutation deliberately removes both static and runtime refusal; +its unawaited-coroutine warning is retained in stderr as an adverse observation, +not accepted as a kill. The required kill is a genuine self-test assertion. +These are finite engineering controls in a trusted interpreter with frozen tests; +this is not hostile-process containment or an exhaustive proof of unittest. +New composed-tree full regression/actual-browser execution remains pending. +Canonical tooling registration should include this amendment and its self-tests, +not relabel historical source receipts as current-source qualification. + + +## Exact code-object identity correction + +The subsequent reviewer found that Python CodeType objects can be structurally +`==` despite distinct identities, including after `co_filename` replacement. +At published parent `8d24c37ff52c012294d71bbcbf08bd24fae4e22a`, set membership +through a frozenset used equality and admitted such a cloned assertion helper. +The new clone regression first failed on that parent. Approved code references +are now retained in a tuple and checked with explicit `is`, never equality or +hash-set membership. All prior controls remain; current counts are31 module +methods /46 combined methods per mode, plus seven grader safeguard mutants. +Evidence is in `evidence/browser-code-identity-20261029/`. Earlier30/45-method +logs and six-mutant observations remain historical at their original source. +This closes the exact-provenance contract defect, not a mathematical proof gap; +no evidence shows earlier actual finite kills or browser outputs were false. diff --git a/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/normal.stderr b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/normal.stderr new file mode 100644 index 000000000..a7da7a216 --- /dev/null +++ b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/normal.stderr @@ -0,0 +1,7 @@ +test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary (tests.test_atlas_focus_geometry.AtlasListFocusGeometry.test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary) ... ok +test_new_selection_after_a_long_read_keeps_its_heading_visible (tests.test_atlas_reader_scroll.AtlasReaderScroll.test_new_selection_after_a_long_read_keeps_its_heading_visible) ... ok + +---------------------------------------------------------------------- +Ran 2 tests in 2.931s + +OK diff --git a/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/normal.stdout b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/normal.stdout new file mode 100644 index 000000000..e69de29bb diff --git a/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/normal/focus.json b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/normal/focus.json new file mode 100644 index 000000000..28ac99fe8 --- /dev/null +++ b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/normal/focus.json @@ -0,0 +1,22 @@ +{ + "first": { + "ring": 7, + "top": 8, + "left": 8, + "right": 7.59375, + "bottom": 388.515625, + "focused": true, + "outline": "solid", + "documentWidth": 390 + }, + "last": { + "ring": 7, + "top": 360.96875, + "left": 8, + "right": 7.59375, + "bottom": 18.046875, + "focused": true, + "outline": "solid", + "documentWidth": 390 + } +} diff --git a/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/normal/reader.json b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/normal/reader.json new file mode 100644 index 000000000..f4c73fc83 --- /dev/null +++ b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/normal/reader.json @@ -0,0 +1,90 @@ +[ + { + "width": 1440, + "browser": "156.0.8078.47", + "initial": { + "scrollTop": 0, + "pageScroll": 0, + "heading": "math.uniform-matrix-cap-lifetime", + "headingTop": 506.59375, + "headingBottom": 532.515625, + "viewport": 1000, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 1440 + }, + "after_read": { + "scrollTop": 0, + "pageScroll": 1812, + "heading": "math.uniform-matrix-cap-lifetime", + "headingTop": -1305.40625, + "headingBottom": -1279.484375, + "viewport": 1000, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 1440 + }, + "new_selection": { + "scrollTop": 0, + "pageScroll": 395, + "heading": "math.d5-component.punctured-pin-proof", + "headingTop": 111.59375, + "headingBottom": 137.515625, + "viewport": 1000, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 1440 + }, + "selection_focus": "atlas-inspector", + "keyboard_source": { + "tag": "A", + "href": "https://github.com/d6g8k5htny-coder/Math-/blob/dbe6c0a1b1a22c7404b0fd173ee0043b98a08cde/reviews/d5_local_collar_20260928/PUNCTURED_PIN_PROOF.md", + "visible": true, + "inside": true + } + }, + { + "width": 390, + "browser": "156.0.8078.47", + "initial": { + "scrollTop": 0, + "pageScroll": 397, + "heading": "math.uniform-matrix-cap-lifetime", + "headingTop": 292.046875, + "headingBottom": 343.890625, + "viewport": 844, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 390 + }, + "after_read": { + "scrollTop": 0, + "pageScroll": 1897, + "heading": "math.uniform-matrix-cap-lifetime", + "headingTop": -1207.953125, + "headingBottom": -1156.109375, + "viewport": 844, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 390 + }, + "new_selection": { + "scrollTop": 0, + "pageScroll": 397, + "heading": "math.d5-component.punctured-pin-proof", + "headingTop": 292.046875, + "headingBottom": 369.8125, + "viewport": 844, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 390 + }, + "selection_focus": "atlas-inspector", + "keyboard_source": { + "tag": "A", + "href": "https://github.com/d6g8k5htny-coder/Math-/blob/dbe6c0a1b1a22c7404b0fd173ee0043b98a08cde/reviews/d5_local_collar_20260928/PUNCTURED_PIN_PROOF.md", + "visible": true, + "inside": true + } + } +] diff --git a/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/normal/result.json b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/normal/result.json new file mode 100644 index 000000000..31784d119 --- /dev/null +++ b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/normal/result.json @@ -0,0 +1,46 @@ +{ + "errors": 0, + "expected_failures": 0, + "expected_ids": [ + "tests.test_atlas_focus_geometry.AtlasListFocusGeometry.test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary", + "tests.test_atlas_reader_scroll.AtlasReaderScroll.test_new_selection_after_a_long_read_keeps_its_heading_visible" + ], + "failures": 0, + "passed": true, + "runtime": { + "browsers": [ + { + "module": "tests.test_atlas_focus_geometry", + "path": "/opt/google/chrome/google-chrome", + "sha256": "aea09d69ce7f24d5901f6bfb15dd44d0c856e793e0a498f8d8393ec7d2c308ec", + "version": "Google Chrome 156.0.8078.47" + }, + { + "module": "tests.test_atlas_reader_scroll", + "path": "/opt/google/chrome/google-chrome", + "sha256": "aea09d69ce7f24d5901f6bfb15dd44d0c856e793e0a498f8d8393ec7d2c308ec", + "version": "Google Chrome 156.0.8078.47" + } + ], + "optimize": 0, + "playwright": "1.62.0", + "python": "3.12.15 (main, Oct 1 2026, 02:35:59) [GCC 13.3.0]" + }, + "schema": "atlas-browser-regressions/1", + "scientific_effect": "NONE", + "skips": [], + "source_after": { + "commit": "8bd88e5952e4a67ff394e1cab3c272e7ace2f961", + "tree": "c9e3a5656fc95feef3bd360af3ea393846afd1e8" + }, + "source_before": { + "commit": "8bd88e5952e4a67ff394e1cab3c272e7ace2f961", + "tree": "c9e3a5656fc95feef3bd360af3ea393846afd1e8" + }, + "successful_ids": [ + "tests.test_atlas_focus_geometry.AtlasListFocusGeometry.test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary", + "tests.test_atlas_reader_scroll.AtlasReaderScroll.test_new_selection_after_a_long_read_keeps_its_heading_visible" + ], + "tests_run": 2, + "unexpected_successes": 0 +} diff --git a/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/optimized.stderr b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/optimized.stderr new file mode 100644 index 000000000..117057076 --- /dev/null +++ b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/optimized.stderr @@ -0,0 +1,7 @@ +test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary (tests.test_atlas_focus_geometry.AtlasListFocusGeometry.test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary) ... ok +test_new_selection_after_a_long_read_keeps_its_heading_visible (tests.test_atlas_reader_scroll.AtlasReaderScroll.test_new_selection_after_a_long_read_keeps_its_heading_visible) ... ok + +---------------------------------------------------------------------- +Ran 2 tests in 2.856s + +OK diff --git a/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/optimized.stdout b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/optimized.stdout new file mode 100644 index 000000000..e69de29bb diff --git a/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/optimized/focus.json b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/optimized/focus.json new file mode 100644 index 000000000..28ac99fe8 --- /dev/null +++ b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/optimized/focus.json @@ -0,0 +1,22 @@ +{ + "first": { + "ring": 7, + "top": 8, + "left": 8, + "right": 7.59375, + "bottom": 388.515625, + "focused": true, + "outline": "solid", + "documentWidth": 390 + }, + "last": { + "ring": 7, + "top": 360.96875, + "left": 8, + "right": 7.59375, + "bottom": 18.046875, + "focused": true, + "outline": "solid", + "documentWidth": 390 + } +} diff --git a/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/optimized/reader.json b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/optimized/reader.json new file mode 100644 index 000000000..f4c73fc83 --- /dev/null +++ b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/optimized/reader.json @@ -0,0 +1,90 @@ +[ + { + "width": 1440, + "browser": "156.0.8078.47", + "initial": { + "scrollTop": 0, + "pageScroll": 0, + "heading": "math.uniform-matrix-cap-lifetime", + "headingTop": 506.59375, + "headingBottom": 532.515625, + "viewport": 1000, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 1440 + }, + "after_read": { + "scrollTop": 0, + "pageScroll": 1812, + "heading": "math.uniform-matrix-cap-lifetime", + "headingTop": -1305.40625, + "headingBottom": -1279.484375, + "viewport": 1000, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 1440 + }, + "new_selection": { + "scrollTop": 0, + "pageScroll": 395, + "heading": "math.d5-component.punctured-pin-proof", + "headingTop": 111.59375, + "headingBottom": 137.515625, + "viewport": 1000, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 1440 + }, + "selection_focus": "atlas-inspector", + "keyboard_source": { + "tag": "A", + "href": "https://github.com/d6g8k5htny-coder/Math-/blob/dbe6c0a1b1a22c7404b0fd173ee0043b98a08cde/reviews/d5_local_collar_20260928/PUNCTURED_PIN_PROOF.md", + "visible": true, + "inside": true + } + }, + { + "width": 390, + "browser": "156.0.8078.47", + "initial": { + "scrollTop": 0, + "pageScroll": 397, + "heading": "math.uniform-matrix-cap-lifetime", + "headingTop": 292.046875, + "headingBottom": 343.890625, + "viewport": 844, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 390 + }, + "after_read": { + "scrollTop": 0, + "pageScroll": 1897, + "heading": "math.uniform-matrix-cap-lifetime", + "headingTop": -1207.953125, + "headingBottom": -1156.109375, + "viewport": 844, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 390 + }, + "new_selection": { + "scrollTop": 0, + "pageScroll": 397, + "heading": "math.d5-component.punctured-pin-proof", + "headingTop": 292.046875, + "headingBottom": 369.8125, + "viewport": 844, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 390 + }, + "selection_focus": "atlas-inspector", + "keyboard_source": { + "tag": "A", + "href": "https://github.com/d6g8k5htny-coder/Math-/blob/dbe6c0a1b1a22c7404b0fd173ee0043b98a08cde/reviews/d5_local_collar_20260928/PUNCTURED_PIN_PROOF.md", + "visible": true, + "inside": true + } + } +] diff --git a/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/optimized/result.json b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/optimized/result.json new file mode 100644 index 000000000..4d79c0849 --- /dev/null +++ b/docs/cbz/evidence/atlas-browser-20261029/corrected/atlas-regressions/optimized/result.json @@ -0,0 +1,46 @@ +{ + "errors": 0, + "expected_failures": 0, + "expected_ids": [ + "tests.test_atlas_focus_geometry.AtlasListFocusGeometry.test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary", + "tests.test_atlas_reader_scroll.AtlasReaderScroll.test_new_selection_after_a_long_read_keeps_its_heading_visible" + ], + "failures": 0, + "passed": true, + "runtime": { + "browsers": [ + { + "module": "tests.test_atlas_focus_geometry", + "path": "/opt/google/chrome/google-chrome", + "sha256": "aea09d69ce7f24d5901f6bfb15dd44d0c856e793e0a498f8d8393ec7d2c308ec", + "version": "Google Chrome 156.0.8078.47" + }, + { + "module": "tests.test_atlas_reader_scroll", + "path": "/opt/google/chrome/google-chrome", + "sha256": "aea09d69ce7f24d5901f6bfb15dd44d0c856e793e0a498f8d8393ec7d2c308ec", + "version": "Google Chrome 156.0.8078.47" + } + ], + "optimize": 1, + "playwright": "1.62.0", + "python": "3.12.15 (main, Oct 1 2026, 02:35:59) [GCC 13.3.0]" + }, + "schema": "atlas-browser-regressions/1", + "scientific_effect": "NONE", + "skips": [], + "source_after": { + "commit": "8bd88e5952e4a67ff394e1cab3c272e7ace2f961", + "tree": "c9e3a5656fc95feef3bd360af3ea393846afd1e8" + }, + "source_before": { + "commit": "8bd88e5952e4a67ff394e1cab3c272e7ace2f961", + "tree": "c9e3a5656fc95feef3bd360af3ea393846afd1e8" + }, + "successful_ids": [ + "tests.test_atlas_focus_geometry.AtlasListFocusGeometry.test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary", + "tests.test_atlas_reader_scroll.AtlasReaderScroll.test_new_selection_after_a_long_read_keeps_its_heading_visible" + ], + "tests_run": 2, + "unexpected_successes": 0 +} diff --git a/docs/cbz/evidence/atlas-browser-20261029/manifest.json b/docs/cbz/evidence/atlas-browser-20261029/manifest.json new file mode 100644 index 000000000..d21c4fdf5 --- /dev/null +++ b/docs/cbz/evidence/atlas-browser-20261029/manifest.json @@ -0,0 +1,185 @@ +{ + "scope": "Engineering execution evidence only; scientific effect NONE", + "observed_utc": "2026-10-29T20:54:00Z", + "head": "c0205ca9e8683b2847008f712582e183f723e186", + "tested_merge": "8bd88e5952e4a67ff394e1cab3c272e7ace2f961", + "tested_tree": "c9e3a5656fc95feef3bd360af3ea393846afd1e8", + "artifacts": [ + { + "label": "original", + "id": 12924209403, + "run_id": 40936994120, + "sha256": "e6ac7e329f63c55dc1188f6dddeacfb3a72b1f60a2327dcaac657cdaee0b3f8a", + "bytes": 35324052, + "fresh_download": true, + "members": [ + { + "path": "atlas-regressions/normal/reader-1440.png", + "bytes": 186847, + "sha256": "db000dfd63ac43e9291c0b003a84c8dd483f150a0b859b97c021867c37fd1c66" + }, + { + "path": "atlas-regressions/normal/reader-390.png", + "bytes": 67416, + "sha256": "209dff618c217bba4d426ffdb9bab212257715842673137ad88ef89e4ad7c232" + }, + { + "path": "atlas-regressions/normal/reader.json", + "bytes": 2533, + "sha256": "f33119ceb3d3fd265f9a53580e12c4ffa5f192d413054018a529539de3df9224" + }, + { + "path": "atlas-regressions/normal/result.json", + "bytes": 1583, + "sha256": "ca2cd2d20d11bf0751475498660ee4f090decfe663db6d1164c60b3eab82f0fd" + }, + { + "path": "atlas-regressions/normal.stdout", + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "atlas-regressions/optimized/reader-1440.png", + "bytes": 186847, + "sha256": "db000dfd63ac43e9291c0b003a84c8dd483f150a0b859b97c021867c37fd1c66" + }, + { + "path": "atlas-regressions/normal.stderr", + "bytes": 13298, + "sha256": "164319e59c00fd7c7ef22ef82797a06f1689e87c4edfe4c0af52a833897b550f" + }, + { + "path": "atlas-regressions/optimized/reader.json", + "bytes": 2533, + "sha256": "f33119ceb3d3fd265f9a53580e12c4ffa5f192d413054018a529539de3df9224" + }, + { + "path": "atlas-regressions/optimized/result.json", + "bytes": 1583, + "sha256": "c9c1678b16d8daf3b49de9a6f5c0a7e7bb4f331e88a39cc8eb0766bf4b9f598e" + }, + { + "path": "atlas-regressions/optimized/reader-390.png", + "bytes": 67416, + "sha256": "209dff618c217bba4d426ffdb9bab212257715842673137ad88ef89e4ad7c232" + }, + { + "path": "atlas-regressions/optimized.stderr", + "bytes": 13298, + "sha256": "8a31932f11aa28923440776016ee2714029e76f565dcd3383acca1e6e0a5c5b1" + }, + { + "path": "atlas-regressions/optimized.stdout", + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + ] + }, + { + "label": "corrected", + "id": 12931733641, + "run_id": 40950918974, + "sha256": "2bcd2bd3ceec0641c56f52bab6159e141e825a10042d9276bd716e7f12c1df36", + "bytes": 35632644, + "fresh_download": true, + "members": [ + { + "path": "atlas-regressions/normal/focus.json", + "bytes": 369, + "sha256": "6ad5e02568fd7a09921154aaaf5a810762fc8fa64eb83546f36813615e3381b3" + }, + { + "path": "atlas-regressions/normal/focus-last.png", + "bytes": 65203, + "sha256": "3239645ad907ed84a958e928cb62cfaab38c47d9337f9b94ca9d05deb22cdcf1" + }, + { + "path": "atlas-regressions/normal/focus.png", + "bytes": 66316, + "sha256": "56757604985780e5479bd14c5a5afc08531c228f584f45b0af0186c988398405" + }, + { + "path": "atlas-regressions/normal/reader-1440.png", + "bytes": 186847, + "sha256": "db000dfd63ac43e9291c0b003a84c8dd483f150a0b859b97c021867c37fd1c66" + }, + { + "path": "atlas-regressions/normal/reader-390.png", + "bytes": 67416, + "sha256": "209dff618c217bba4d426ffdb9bab212257715842673137ad88ef89e4ad7c232" + }, + { + "path": "atlas-regressions/normal/reader.json", + "bytes": 2533, + "sha256": "f33119ceb3d3fd265f9a53580e12c4ffa5f192d413054018a529539de3df9224" + }, + { + "path": "atlas-regressions/normal/result.json", + "bytes": 1699, + "sha256": "7e477a15521fe7ca29a1baaa9e086cd70ba21c7993cb7cbf84ced863a89c2ae0" + }, + { + "path": "atlas-regressions/normal.stderr", + "bytes": 470, + "sha256": "d4293c7e73f1cac3751c5951dd611ed9daa5baea5dbf6d44857ee43c88cefc44" + }, + { + "path": "atlas-regressions/normal.stdout", + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "atlas-regressions/optimized/focus-last.png", + "bytes": 65203, + "sha256": "3239645ad907ed84a958e928cb62cfaab38c47d9337f9b94ca9d05deb22cdcf1" + }, + { + "path": "atlas-regressions/optimized/focus.json", + "bytes": 369, + "sha256": "6ad5e02568fd7a09921154aaaf5a810762fc8fa64eb83546f36813615e3381b3" + }, + { + "path": "atlas-regressions/optimized/focus.png", + "bytes": 66316, + "sha256": "56757604985780e5479bd14c5a5afc08531c228f584f45b0af0186c988398405" + }, + { + "path": "atlas-regressions/optimized/reader-1440.png", + "bytes": 186847, + "sha256": "db000dfd63ac43e9291c0b003a84c8dd483f150a0b859b97c021867c37fd1c66" + }, + { + "path": "atlas-regressions/optimized/reader-390.png", + "bytes": 67416, + "sha256": "209dff618c217bba4d426ffdb9bab212257715842673137ad88ef89e4ad7c232" + }, + { + "path": "atlas-regressions/optimized/reader.json", + "bytes": 2533, + "sha256": "f33119ceb3d3fd265f9a53580e12c4ffa5f192d413054018a529539de3df9224" + }, + { + "path": "atlas-regressions/optimized/result.json", + "bytes": 1699, + "sha256": "a5a0dff2bfd4c63a359ce6ba5d98134f86dd9ed1f26485498d4e90fe6325acc3" + }, + { + "path": "atlas-regressions/optimized.stderr", + "bytes": 470, + "sha256": "8140940ab0714bd83b3e1cda7accd9235bd48e73e79c5e8294be623217eb74af" + }, + { + "path": "atlas-regressions/optimized.stdout", + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + ] + } + ], + "job_log": { + "job_id": 123718493908, + "sha256": "c27de18ed137b0ea77097013b2130e316ea652b6aa374daad6ad2b8f342d19e5", + "bytes": 1046950, + "source": "GitHub decoded job-log connector; full local log retained, not reproduced in source contribution" + } +} diff --git a/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/normal.stderr b/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/normal.stderr new file mode 100644 index 000000000..aa860eb4b --- /dev/null +++ b/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/normal.stderr @@ -0,0 +1,159 @@ +test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary (tests.test_atlas_focus_geometry.AtlasListFocusGeometry.test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary) ... ERROR +test_new_selection_after_a_long_read_keeps_its_heading_visible (tests.test_atlas_reader_scroll.AtlasReaderScroll.test_new_selection_after_a_long_read_keeps_its_heading_visible) ... ok + +====================================================================== +ERROR: test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary (tests.test_atlas_focus_geometry.AtlasListFocusGeometry.test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/home/runner/work/main/main/tests/test_atlas_focus_geometry.py", line 34, in test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary + browser = playwright.chromium.launch(executable_path=CHROMIUM, headless=True, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/playwright/sync_api/_generated.py", line 16702, in launch + self._sync( + File "/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/playwright/_impl/_sync_base.py", line 115, in _sync + return task.result() + ^^^^^^^^^^^^^ + File "/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/playwright/_impl/_browser_type.py", line 97, in launch + await self._channel.send( + File "/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/playwright/_impl/_connection.py", line 76, in send + return await self._connection.wrap_api_call( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/playwright/_impl/_connection.py", line 632, in wrap_api_call + raise rewrite_error(error, f"{parsed_st['apiName']}: {error}") from None +playwright._impl._errors.TargetClosedError: BrowserType.launch: Target page, context or browser has been closed +Browser logs: +Chromium sandboxing failed! +================================ +To avoid the sandboxing issue, do either of the following: + - (preferred): Configure your environment to support sandboxing + - (alternative): Launch Chromium without sandbox using 'chromiumSandbox: false' option +================================ + +Call log: + - /usr/bin/chromium --disable-field-trial-config --disable-background-networking --disable-background-timer-throttling --disable-backgrounding-occluded-windows --disable-back-forward-cache --disable-breakpad --disable-client-side-phishing-detection --disable-component-extensions-with-background-pages --disable-component-update --no-default-browser-check --disable-default-apps --disable-dev-shm-usage --disable-edgeupdater --disable-extensions --disable-features=AvoidUnnecessaryBeforeUnloadCheckSync,BoundaryEventDispatchTracksNodeRemoval,DestroyProfileOnBrowserClose,DialMediaRouteProvider,GlobalMediaControls,HttpsUpgrades,LensOverlay,MediaRouter,PaintHolding,ThirdPartyStoragePartitioning,BlockOriginHeaderModificationOnRedirect,Translate,AutoDeElevate,OptimizationHints,msForceBrowserSignIn,msEdgeUpdateLaunchServicesPreferredVersion --enable-features=CDPScreenshotNewSurface --allow-pre-commit-input --disable-hang-monitor --disable-ipc-flooding-protection --disable-popup-blocking --disable-prompt-on-repost --disable-renderer-backgrounding --disable-updater-scheduler --force-color-profile=srgb --metrics-recording-only --no-first-run --password-store=basic --use-mock-keychain --no-service-autorun --export-tagged-pdf --disable-search-engine-choice-screen --unsafely-disable-devtools-self-xss-warnings --edge-skip-compat-layer-relaunch --disable-infobars --disable-search-engine-choice-screen --disable-sync --enable-unsafe-swiftshader --headless --hide-scrollbars --mute-audio --blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4 --user-data-dir=/tmp/playwright_chromiumdev_profile-MJ2NyG --remote-debugging-pipe --no-startup-window + - pid=11152 + - [pid=11152][err] [11152:11152:1029/190925.592728:FATAL:content/browser/zygote_host/zygote_host_impl_linux.cc:130] No usable sandbox! If you are running on Ubuntu 23.10+ or another Linux distro that has disabled unprivileged user namespaces with AppArmor, see https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md. Otherwise see https://chromium.googlesource.com/chromium/src/+/main/docs/linux/suid_sandbox_development.md for more information on developing with the (older) SUID sandbox. If you want to live dangerously and need an immediate workaround, you can try using --no-sandbox. + - [pid=11152][err] #0 0x556ee1e1c4d2 base::debug::CollectStackTrace() + - [pid=11152][err] #1 0x556ee1e06afe base::debug::StackTrace::StackTrace() + - [pid=11152][err] #2 0x556ee1d3962c logging::LogMessage::Flush() + - [pid=11152][err] #3 0x556ee1d3a1c9 logging::LogMessageFatal::~LogMessageFatal() + - [pid=11152][err] #4 0x556edea7f0d2 content::ZygoteHostImpl::Init() + - [pid=11152][err] #5 0x556ee0d18b69 content::ContentMainRunnerImpl::Initialize() + - [pid=11152][err] #6 0x556ee0d1620c content::RunContentProcess() + - [pid=11152][err] #7 0x556ee0d16657 content::ContentMain() + - [pid=11152][err] #8 0x556edac4337d ChromeMain + - [pid=11152][err] #9 0x7fdba562a1ca (/usr/lib/x86_64-linux-gnu/libc.so.6+0x2a1c9) + - [pid=11152][err] #10 0x7fdba562a28b __libc_start_main + - [pid=11152][err] #11 0x556edac4302a _start + - [pid=11152][err] Crash keys: + - [pid=11152][err] "switch-46" = "--use-angle=swiftshader-webgl" + - [pid=11152][err] "switch-45" = "--ozone-override-screen-size=800,600" + - [pid=11152][err] "switch-44" = "--ozone-platform=headless" + - [pid=11152][err] "switch-43" = "--noerrdialogs" + - [pid=11152][err] "switch-42" = "--no-startup-window" + - [pid=11152][err] "switch-41" = "--remote-debugging-pipe" + - [pid=11152][err] "switch-40" = "--user-data-dir=/tmp/playwright_chromiumdev_profile-MJ2NyG" + - [pid=11152][err] "switch-39" = "--blink-settings=primaryHoverType=2,availableHoverTypes=2,primar" + - [pid=11152][err] "switch-38" = "--mute-audio" + - [pid=11152][err] "switch-37" = "--hide-scrollbars" + - [pid=11152][err] "switch-36" = "--headless" + - [pid=11152][err] "switch-35" = "--enable-unsafe-swiftshader" + - [pid=11152][err] "switch-34" = "--disable-sync" + - [pid=11152][err] "switch-33" = "--disable-search-engine-choice-screen" + - [pid=11152][err] "switch-32" = "--disable-infobars" + - [pid=11152][err] "switch-31" = "--edge-skip-compat-layer-relaunch" + - [pid=11152][err] "switch-30" = "--unsafely-disable-devtools-self-xss-warnings" + - [pid=11152][err] "switch-29" = "--disable-search-engine-choice-screen" + - [pid=11152][err] "switch-28" = "--export-tagged-pdf" + - [pid=11152][err] "switch-27" = "--no-service-autorun" + - [pid=11152][err] "switch-26" = "--use-mock-keychain" + - [pid=11152][err] "switch-25" = "--password-store=basic" + - [pid=11152][err] "switch-24" = "--no-first-run" + - [pid=11152][err] "switch-23" = "--metrics-recording-only" + - [pid=11152][err] "switch-22" = "--force-color-profile=srgb" + - [pid=11152][err] "switch-21" = "--disable-updater-scheduler" + - [pid=11152][err] "switch-20" = "--disable-renderer-backgrounding" + - [pid=11152][err] "switch-19" = "--disable-prompt-on-repost" + - [pid=11152][err] "switch-18" = "--disable-popup-blocking" + - [pid=11152][err] "switch-17" = "--disable-ipc-flooding-protection" + - [pid=11152][err] "switch-16" = "--disable-hang-monitor" + - [pid=11152][err] "switch-15" = "--allow-pre-commit-input" + - [pid=11152][err] "switch-14" = "--disable-extensions" + - [pid=11152][err] "switch-13" = "--disable-edgeupdater" + - [pid=11152][err] "switch-12" = "--disable-dev-shm-usage" + - [pid=11152][err] "switch-11" = "--disable-default-apps" + - [pid=11152][err] "switch-10" = "--no-default-browser-check" + - [pid=11152][err] "switch-9" = "--disable-component-update" + - [pid=11152][err] "switch-8" = "--disable-component-extensions-with-background-pages" + - [pid=11152][err] "switch-7" = "--disable-client-side-phishing-detection" + - [pid=11152][err] "switch-6" = "--disable-breakpad" + - [pid=11152][err] "switch-5" = "--disable-back-forward-cache" + - [pid=11152][err] "switch-4" = "--disable-backgrounding-occluded-windows" + - [pid=11152][err] "switch-3" = "--disable-background-timer-throttling" + - [pid=11152][err] "switch-2" = "--disable-background-networking" + - [pid=11152][err] "switch-1" = "--disable-field-trial-config" + - [pid=11152][err] "num-switches" = "48" + - [pid=11152][err] "commandline-disabled-feature-16" = "msEdgeUpdateLaunchServicesPreferredVersion" + - [pid=11152][err] "commandline-disabled-feature-15" = "msForceBrowserSignIn" + - [pid=11152][err] "commandline-disabled-feature-14" = "OptimizationHints" + - [pid=11152][err] "commandline-disabled-feature-13" = "AutoDeElevate" + - [pid=11152][err] "commandline-disabled-feature-12" = "Translate" + - [pid=11152][err] "commandline-disabled-feature-11" = "BlockOriginHeaderModificationOnRedirect" + - [pid=11152][err] "commandline-disabled-feature-10" = "ThirdPartyStoragePartitioning" + - [pid=11152][err] "commandline-disabled-feature-9" = "PaintHolding" + - [pid=11152][err] "commandline-disabled-feature-8" = "MediaRouter" + - [pid=11152][err] "commandline-disabled-feature-7" = "LensOverlay" + - [pid=11152][err] "commandline-disabled-feature-6" = "HttpsUpgrades" + - [pid=11152][err] "commandline-disabled-feature-5" = "GlobalMediaControls" + - [pid=11152][err] "commandline-disabled-feature-4" = "DialMediaRouteProvider" + - [pid=11152][err] "commandline-disabled-feature-3" = "DestroyProfileOnBrowserClose" + - [pid=11152][err] "commandline-disabled-feature-2" = "BoundaryEventDispatchTracksNodeRemoval" + - [pid=11152][err] "commandline-disabled-feature-1" = "AvoidUnnecessaryBeforeUnloadCheckSync" + - [pid=11152][err] "commandline-enabled-feature-1" = "CDPScreenshotNewSurface" + - [pid=11152][err] "osarch" = "x86_64" + - [pid=11152][err] "pid" = "11152" + - [pid=11152][err] "ptype" = "browser" + - [pid=11152][err] + - [pid=11152][err] [1029/190925.686238:ERROR:third_party/crashpad/crashpad/util/file/file_io_posix.cc:145] open /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq: No such file or directory (2) + - [pid=11152][err] [1029/190925.686288:ERROR:third_party/crashpad/crashpad/util/file/file_io_posix.cc:145] open /sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq: No such file or directory (2) + - [pid=11152][err] Received signal 6 + - [pid=11152][err] #0 0x556ee1e1c4d2 base::debug::CollectStackTrace() + - [pid=11152][err] #1 0x556ee1e06afe base::debug::StackTrace::StackTrace() + - [pid=11152][err] #2 0x556ee1e1bf38 base::debug::(anonymous namespace)::StackDumpSignalHandler() + - [pid=11152][err] #3 0x7fdba5645330 (/usr/lib/x86_64-linux-gnu/libc.so.6+0x4532f) + - [pid=11152][err] #4 0x7fdba569ec0c pthread_kill + - [pid=11152][err] #5 0x7fdba564527e gsignal + - [pid=11152][err] #6 0x7fdba56288ff abort + - [pid=11152][err] #7 0x556ee1df5a35 base::debug::BreakDebuggerAsyncSafe() + - [pid=11152][err] #8 0x556ee1d3a173 base::RepeatingCallback<>::Run() + - [pid=11152][err] #9 0x556ee1d3a035 logging::LogMessage::HandleFatal() + - [pid=11152][err] #10 0x556ee1d39c27 logging::LogMessage::Flush() + - [pid=11152][err] #11 0x556ee1d3a1c9 logging::LogMessageFatal::~LogMessageFatal() + - [pid=11152][err] #12 0x556edea7f0d2 content::ZygoteHostImpl::Init() + - [pid=11152][err] #13 0x556ee0d18b69 content::ContentMainRunnerImpl::Initialize() + - [pid=11152][err] #14 0x556ee0d1620c content::RunContentProcess() + - [pid=11152][err] #15 0x556ee0d16657 content::ContentMain() + - [pid=11152][err] #16 0x556edac4337d ChromeMain + - [pid=11152][err] #17 0x7fdba562a1ca (/usr/lib/x86_64-linux-gnu/libc.so.6+0x2a1c9) + - [pid=11152][err] #18 0x7fdba562a28b __libc_start_main + - [pid=11152][err] #19 0x556edac4302a _start + - [pid=11152][err] r8: 00002b8000078265 r9: 0000000000000f1e r10: 0000000000000008 r11: 0000000000000246 + - [pid=11152][err] r12: 0000000000000006 r13: 0000000000001183 r14: 0000000000000016 r15: 00007ffddd9253a0 + - [pid=11152][err] di: 0000000000002b90 si: 0000000000002b90 bp: 00007ffddd925250 bx: 0000000000002b90 + - [pid=11152][err] dx: 0000000000000006 ax: 0000000000000000 cx: 00007fdba569ec0c sp: 00007ffddd925210 + - [pid=11152][err] ip: 00007fdba569ec0c efl: 0000000000000246 cgf: 002b000000000033 erf: 0000000000000000 + - [pid=11152][err] trp: 0000000000000000 msk: 0000000000000000 cr2: 0000000000000000 + - [pid=11152][err] [end of stack trace] + - [pid=11152] + - [pid=11152] + - [pid=11152] + - [pid=11152] + - [pid=11152] starting temporary directories cleanup + - [pid=11152] finished temporary directories cleanup + - [pid=11152] + + +---------------------------------------------------------------------- +Ran 2 tests in 4.114s + +FAILED (errors=1) diff --git a/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/normal.stdout b/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/normal.stdout new file mode 100644 index 000000000..e69de29bb diff --git a/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/normal/reader.json b/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/normal/reader.json new file mode 100644 index 000000000..f4c73fc83 --- /dev/null +++ b/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/normal/reader.json @@ -0,0 +1,90 @@ +[ + { + "width": 1440, + "browser": "156.0.8078.47", + "initial": { + "scrollTop": 0, + "pageScroll": 0, + "heading": "math.uniform-matrix-cap-lifetime", + "headingTop": 506.59375, + "headingBottom": 532.515625, + "viewport": 1000, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 1440 + }, + "after_read": { + "scrollTop": 0, + "pageScroll": 1812, + "heading": "math.uniform-matrix-cap-lifetime", + "headingTop": -1305.40625, + "headingBottom": -1279.484375, + "viewport": 1000, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 1440 + }, + "new_selection": { + "scrollTop": 0, + "pageScroll": 395, + "heading": "math.d5-component.punctured-pin-proof", + "headingTop": 111.59375, + "headingBottom": 137.515625, + "viewport": 1000, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 1440 + }, + "selection_focus": "atlas-inspector", + "keyboard_source": { + "tag": "A", + "href": "https://github.com/d6g8k5htny-coder/Math-/blob/dbe6c0a1b1a22c7404b0fd173ee0043b98a08cde/reviews/d5_local_collar_20260928/PUNCTURED_PIN_PROOF.md", + "visible": true, + "inside": true + } + }, + { + "width": 390, + "browser": "156.0.8078.47", + "initial": { + "scrollTop": 0, + "pageScroll": 397, + "heading": "math.uniform-matrix-cap-lifetime", + "headingTop": 292.046875, + "headingBottom": 343.890625, + "viewport": 844, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 390 + }, + "after_read": { + "scrollTop": 0, + "pageScroll": 1897, + "heading": "math.uniform-matrix-cap-lifetime", + "headingTop": -1207.953125, + "headingBottom": -1156.109375, + "viewport": 844, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 390 + }, + "new_selection": { + "scrollTop": 0, + "pageScroll": 397, + "heading": "math.d5-component.punctured-pin-proof", + "headingTop": 292.046875, + "headingBottom": 369.8125, + "viewport": 844, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 390 + }, + "selection_focus": "atlas-inspector", + "keyboard_source": { + "tag": "A", + "href": "https://github.com/d6g8k5htny-coder/Math-/blob/dbe6c0a1b1a22c7404b0fd173ee0043b98a08cde/reviews/d5_local_collar_20260928/PUNCTURED_PIN_PROOF.md", + "visible": true, + "inside": true + } + } +] diff --git a/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/normal/result.json b/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/normal/result.json new file mode 100644 index 000000000..51a369651 --- /dev/null +++ b/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/normal/result.json @@ -0,0 +1,45 @@ +{ + "errors": 1, + "expected_failures": 0, + "expected_ids": [ + "tests.test_atlas_focus_geometry.AtlasListFocusGeometry.test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary", + "tests.test_atlas_reader_scroll.AtlasReaderScroll.test_new_selection_after_a_long_read_keeps_its_heading_visible" + ], + "failures": 0, + "passed": false, + "runtime": { + "browsers": [ + { + "module": "tests.test_atlas_focus_geometry", + "path": "/usr/local/share/chromium/chrome-linux/chrome", + "sha256": "c97b58644d050f2affcb5132b7cb5690162ad459dfbbbaca1b3ef85cadcf2f21", + "version": "Chromium 156.0.8078.0" + }, + { + "module": "tests.test_atlas_reader_scroll", + "path": "/opt/google/chrome/google-chrome", + "sha256": "aea09d69ce7f24d5901f6bfb15dd44d0c856e793e0a498f8d8393ec7d2c308ec", + "version": "Google Chrome 156.0.8078.47" + } + ], + "optimize": 0, + "playwright": "1.62.0", + "python": "3.12.15 (main, Oct 1 2026, 02:35:59) [GCC 13.3.0]" + }, + "schema": "atlas-browser-regressions/1", + "scientific_effect": "NONE", + "skips": [], + "source_after": { + "commit": "6abed714e07816e74be07ec5cd3fd6f94e6d046e", + "tree": "535097c43edf631ede9b8ee21b1697b497b550c0" + }, + "source_before": { + "commit": "6abed714e07816e74be07ec5cd3fd6f94e6d046e", + "tree": "535097c43edf631ede9b8ee21b1697b497b550c0" + }, + "successful_ids": [ + "tests.test_atlas_reader_scroll.AtlasReaderScroll.test_new_selection_after_a_long_read_keeps_its_heading_visible" + ], + "tests_run": 2, + "unexpected_successes": 0 +} diff --git a/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/optimized.stderr b/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/optimized.stderr new file mode 100644 index 000000000..e4cee0436 --- /dev/null +++ b/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/optimized.stderr @@ -0,0 +1,159 @@ +test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary (tests.test_atlas_focus_geometry.AtlasListFocusGeometry.test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary) ... ERROR +test_new_selection_after_a_long_read_keeps_its_heading_visible (tests.test_atlas_reader_scroll.AtlasReaderScroll.test_new_selection_after_a_long_read_keeps_its_heading_visible) ... ok + +====================================================================== +ERROR: test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary (tests.test_atlas_focus_geometry.AtlasListFocusGeometry.test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/home/runner/work/main/main/tests/test_atlas_focus_geometry.py", line 34, in test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary + browser = playwright.chromium.launch(executable_path=CHROMIUM, headless=True, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/playwright/sync_api/_generated.py", line 16702, in launch + self._sync( + File "/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/playwright/_impl/_sync_base.py", line 115, in _sync + return task.result() + ^^^^^^^^^^^^^ + File "/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/playwright/_impl/_browser_type.py", line 97, in launch + await self._channel.send( + File "/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/playwright/_impl/_connection.py", line 76, in send + return await self._connection.wrap_api_call( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/playwright/_impl/_connection.py", line 632, in wrap_api_call + raise rewrite_error(error, f"{parsed_st['apiName']}: {error}") from None +playwright._impl._errors.TargetClosedError: BrowserType.launch: Target page, context or browser has been closed +Browser logs: +Chromium sandboxing failed! +================================ +To avoid the sandboxing issue, do either of the following: + - (preferred): Configure your environment to support sandboxing + - (alternative): Launch Chromium without sandbox using 'chromiumSandbox: false' option +================================ + +Call log: + - /usr/bin/chromium --disable-field-trial-config --disable-background-networking --disable-background-timer-throttling --disable-backgrounding-occluded-windows --disable-back-forward-cache --disable-breakpad --disable-client-side-phishing-detection --disable-component-extensions-with-background-pages --disable-component-update --no-default-browser-check --disable-default-apps --disable-dev-shm-usage --disable-edgeupdater --disable-extensions --disable-features=AvoidUnnecessaryBeforeUnloadCheckSync,BoundaryEventDispatchTracksNodeRemoval,DestroyProfileOnBrowserClose,DialMediaRouteProvider,GlobalMediaControls,HttpsUpgrades,LensOverlay,MediaRouter,PaintHolding,ThirdPartyStoragePartitioning,BlockOriginHeaderModificationOnRedirect,Translate,AutoDeElevate,OptimizationHints,msForceBrowserSignIn,msEdgeUpdateLaunchServicesPreferredVersion --enable-features=CDPScreenshotNewSurface --allow-pre-commit-input --disable-hang-monitor --disable-ipc-flooding-protection --disable-popup-blocking --disable-prompt-on-repost --disable-renderer-backgrounding --disable-updater-scheduler --force-color-profile=srgb --metrics-recording-only --no-first-run --password-store=basic --use-mock-keychain --no-service-autorun --export-tagged-pdf --disable-search-engine-choice-screen --unsafely-disable-devtools-self-xss-warnings --edge-skip-compat-layer-relaunch --disable-infobars --disable-search-engine-choice-screen --disable-sync --enable-unsafe-swiftshader --headless --hide-scrollbars --mute-audio --blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4 --user-data-dir=/tmp/playwright_chromiumdev_profile-ixp9VQ --remote-debugging-pipe --no-startup-window + - pid=11457 + - [pid=11457][err] [11457:11457:1029/190931.375776:FATAL:content/browser/zygote_host/zygote_host_impl_linux.cc:130] No usable sandbox! If you are running on Ubuntu 23.10+ or another Linux distro that has disabled unprivileged user namespaces with AppArmor, see https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md. Otherwise see https://chromium.googlesource.com/chromium/src/+/main/docs/linux/suid_sandbox_development.md for more information on developing with the (older) SUID sandbox. If you want to live dangerously and need an immediate workaround, you can try using --no-sandbox. + - [pid=11457][err] #0 0x55ee2d5664d2 base::debug::CollectStackTrace() + - [pid=11457][err] #1 0x55ee2d550afe base::debug::StackTrace::StackTrace() + - [pid=11457][err] #2 0x55ee2d48362c logging::LogMessage::Flush() + - [pid=11457][err] #3 0x55ee2d4841c9 logging::LogMessageFatal::~LogMessageFatal() + - [pid=11457][err] #4 0x55ee2a1c90d2 content::ZygoteHostImpl::Init() + - [pid=11457][err] #5 0x55ee2c462b69 content::ContentMainRunnerImpl::Initialize() + - [pid=11457][err] #6 0x55ee2c46020c content::RunContentProcess() + - [pid=11457][err] #7 0x55ee2c460657 content::ContentMain() + - [pid=11457][err] #8 0x55ee2638d37d ChromeMain + - [pid=11457][err] #9 0x7fcc37a2a1ca (/usr/lib/x86_64-linux-gnu/libc.so.6+0x2a1c9) + - [pid=11457][err] #10 0x7fcc37a2a28b __libc_start_main + - [pid=11457][err] #11 0x55ee2638d02a _start + - [pid=11457][err] Crash keys: + - [pid=11457][err] "switch-46" = "--use-angle=swiftshader-webgl" + - [pid=11457][err] "switch-45" = "--ozone-override-screen-size=800,600" + - [pid=11457][err] "switch-44" = "--ozone-platform=headless" + - [pid=11457][err] "switch-43" = "--noerrdialogs" + - [pid=11457][err] "switch-42" = "--no-startup-window" + - [pid=11457][err] "switch-41" = "--remote-debugging-pipe" + - [pid=11457][err] "switch-40" = "--user-data-dir=/tmp/playwright_chromiumdev_profile-ixp9VQ" + - [pid=11457][err] "switch-39" = "--blink-settings=primaryHoverType=2,availableHoverTypes=2,primar" + - [pid=11457][err] "switch-38" = "--mute-audio" + - [pid=11457][err] "switch-37" = "--hide-scrollbars" + - [pid=11457][err] "switch-36" = "--headless" + - [pid=11457][err] "switch-35" = "--enable-unsafe-swiftshader" + - [pid=11457][err] "switch-34" = "--disable-sync" + - [pid=11457][err] "switch-33" = "--disable-search-engine-choice-screen" + - [pid=11457][err] "switch-32" = "--disable-infobars" + - [pid=11457][err] "switch-31" = "--edge-skip-compat-layer-relaunch" + - [pid=11457][err] "switch-30" = "--unsafely-disable-devtools-self-xss-warnings" + - [pid=11457][err] "switch-29" = "--disable-search-engine-choice-screen" + - [pid=11457][err] "switch-28" = "--export-tagged-pdf" + - [pid=11457][err] "switch-27" = "--no-service-autorun" + - [pid=11457][err] "switch-26" = "--use-mock-keychain" + - [pid=11457][err] "switch-25" = "--password-store=basic" + - [pid=11457][err] "switch-24" = "--no-first-run" + - [pid=11457][err] "switch-23" = "--metrics-recording-only" + - [pid=11457][err] "switch-22" = "--force-color-profile=srgb" + - [pid=11457][err] "switch-21" = "--disable-updater-scheduler" + - [pid=11457][err] "switch-20" = "--disable-renderer-backgrounding" + - [pid=11457][err] "switch-19" = "--disable-prompt-on-repost" + - [pid=11457][err] "switch-18" = "--disable-popup-blocking" + - [pid=11457][err] "switch-17" = "--disable-ipc-flooding-protection" + - [pid=11457][err] "switch-16" = "--disable-hang-monitor" + - [pid=11457][err] "switch-15" = "--allow-pre-commit-input" + - [pid=11457][err] "switch-14" = "--disable-extensions" + - [pid=11457][err] "switch-13" = "--disable-edgeupdater" + - [pid=11457][err] "switch-12" = "--disable-dev-shm-usage" + - [pid=11457][err] "switch-11" = "--disable-default-apps" + - [pid=11457][err] "switch-10" = "--no-default-browser-check" + - [pid=11457][err] "switch-9" = "--disable-component-update" + - [pid=11457][err] "switch-8" = "--disable-component-extensions-with-background-pages" + - [pid=11457][err] "switch-7" = "--disable-client-side-phishing-detection" + - [pid=11457][err] "switch-6" = "--disable-breakpad" + - [pid=11457][err] "switch-5" = "--disable-back-forward-cache" + - [pid=11457][err] "switch-4" = "--disable-backgrounding-occluded-windows" + - [pid=11457][err] "switch-3" = "--disable-background-timer-throttling" + - [pid=11457][err] "switch-2" = "--disable-background-networking" + - [pid=11457][err] "switch-1" = "--disable-field-trial-config" + - [pid=11457][err] "num-switches" = "48" + - [pid=11457][err] "commandline-disabled-feature-16" = "msEdgeUpdateLaunchServicesPreferredVersion" + - [pid=11457][err] "commandline-disabled-feature-15" = "msForceBrowserSignIn" + - [pid=11457][err] "commandline-disabled-feature-14" = "OptimizationHints" + - [pid=11457][err] "commandline-disabled-feature-13" = "AutoDeElevate" + - [pid=11457][err] "commandline-disabled-feature-12" = "Translate" + - [pid=11457][err] "commandline-disabled-feature-11" = "BlockOriginHeaderModificationOnRedirect" + - [pid=11457][err] "commandline-disabled-feature-10" = "ThirdPartyStoragePartitioning" + - [pid=11457][err] "commandline-disabled-feature-9" = "PaintHolding" + - [pid=11457][err] "commandline-disabled-feature-8" = "MediaRouter" + - [pid=11457][err] "commandline-disabled-feature-7" = "LensOverlay" + - [pid=11457][err] "commandline-disabled-feature-6" = "HttpsUpgrades" + - [pid=11457][err] "commandline-disabled-feature-5" = "GlobalMediaControls" + - [pid=11457][err] "commandline-disabled-feature-4" = "DialMediaRouteProvider" + - [pid=11457][err] "commandline-disabled-feature-3" = "DestroyProfileOnBrowserClose" + - [pid=11457][err] "commandline-disabled-feature-2" = "BoundaryEventDispatchTracksNodeRemoval" + - [pid=11457][err] "commandline-disabled-feature-1" = "AvoidUnnecessaryBeforeUnloadCheckSync" + - [pid=11457][err] "commandline-enabled-feature-1" = "CDPScreenshotNewSurface" + - [pid=11457][err] "osarch" = "x86_64" + - [pid=11457][err] "pid" = "11457" + - [pid=11457][err] "ptype" = "browser" + - [pid=11457][err] + - [pid=11457][err] [1029/190931.469486:ERROR:third_party/crashpad/crashpad/util/file/file_io_posix.cc:145] open /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq: No such file or directory (2) + - [pid=11457][err] [1029/190931.469539:ERROR:third_party/crashpad/crashpad/util/file/file_io_posix.cc:145] open /sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq: No such file or directory (2) + - [pid=11457][err] Received signal 6 + - [pid=11457][err] #0 0x55ee2d5664d2 base::debug::CollectStackTrace() + - [pid=11457][err] #1 0x55ee2d550afe base::debug::StackTrace::StackTrace() + - [pid=11457][err] #2 0x55ee2d565f38 base::debug::(anonymous namespace)::StackDumpSignalHandler() + - [pid=11457][err] #3 0x7fcc37a45330 (/usr/lib/x86_64-linux-gnu/libc.so.6+0x4532f) + - [pid=11457][err] #4 0x7fcc37a9ec0c pthread_kill + - [pid=11457][err] #5 0x7fcc37a4527e gsignal + - [pid=11457][err] #6 0x7fcc37a288ff abort + - [pid=11457][err] #7 0x55ee2d53fa35 base::debug::BreakDebuggerAsyncSafe() + - [pid=11457][err] #8 0x55ee2d484173 base::RepeatingCallback<>::Run() + - [pid=11457][err] #9 0x55ee2d484035 logging::LogMessage::HandleFatal() + - [pid=11457][err] #10 0x55ee2d483c27 logging::LogMessage::Flush() + - [pid=11457][err] #11 0x55ee2d4841c9 logging::LogMessageFatal::~LogMessageFatal() + - [pid=11457][err] #12 0x55ee2a1c90d2 content::ZygoteHostImpl::Init() + - [pid=11457][err] #13 0x55ee2c462b69 content::ContentMainRunnerImpl::Initialize() + - [pid=11457][err] #14 0x55ee2c46020c content::RunContentProcess() + - [pid=11457][err] #15 0x55ee2c460657 content::ContentMain() + - [pid=11457][err] #16 0x55ee2638d37d ChromeMain + - [pid=11457][err] #17 0x7fcc37a2a1ca (/usr/lib/x86_64-linux-gnu/libc.so.6+0x2a1c9) + - [pid=11457][err] #18 0x7fcc37a2a28b __libc_start_main + - [pid=11457][err] #19 0x55ee2638d02a _start + - [pid=11457][err] r8: 0000220800078265 r9: 0000000000000f1e r10: 0000000000000008 r11: 0000000000000246 + - [pid=11457][err] r12: 0000000000000006 r13: 0000000000001183 r14: 0000000000000016 r15: 00007fff672861d0 + - [pid=11457][err] di: 0000000000002cc1 si: 0000000000002cc1 bp: 00007fff67286080 bx: 0000000000002cc1 + - [pid=11457][err] dx: 0000000000000006 ax: 0000000000000000 cx: 00007fcc37a9ec0c sp: 00007fff67286040 + - [pid=11457][err] ip: 00007fcc37a9ec0c efl: 0000000000000246 cgf: 002b000000000033 erf: 0000000000000000 + - [pid=11457][err] trp: 0000000000000000 msk: 0000000000000000 cr2: 0000000000000000 + - [pid=11457][err] [end of stack trace] + - [pid=11457] + - [pid=11457] + - [pid=11457] + - [pid=11457] + - [pid=11457] starting temporary directories cleanup + - [pid=11457] finished temporary directories cleanup + - [pid=11457] + + +---------------------------------------------------------------------- +Ran 2 tests in 3.761s + +FAILED (errors=1) diff --git a/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/optimized.stdout b/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/optimized.stdout new file mode 100644 index 000000000..e69de29bb diff --git a/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/optimized/reader.json b/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/optimized/reader.json new file mode 100644 index 000000000..f4c73fc83 --- /dev/null +++ b/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/optimized/reader.json @@ -0,0 +1,90 @@ +[ + { + "width": 1440, + "browser": "156.0.8078.47", + "initial": { + "scrollTop": 0, + "pageScroll": 0, + "heading": "math.uniform-matrix-cap-lifetime", + "headingTop": 506.59375, + "headingBottom": 532.515625, + "viewport": 1000, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 1440 + }, + "after_read": { + "scrollTop": 0, + "pageScroll": 1812, + "heading": "math.uniform-matrix-cap-lifetime", + "headingTop": -1305.40625, + "headingBottom": -1279.484375, + "viewport": 1000, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 1440 + }, + "new_selection": { + "scrollTop": 0, + "pageScroll": 395, + "heading": "math.d5-component.punctured-pin-proof", + "headingTop": 111.59375, + "headingBottom": 137.515625, + "viewport": 1000, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 1440 + }, + "selection_focus": "atlas-inspector", + "keyboard_source": { + "tag": "A", + "href": "https://github.com/d6g8k5htny-coder/Math-/blob/dbe6c0a1b1a22c7404b0fd173ee0043b98a08cde/reviews/d5_local_collar_20260928/PUNCTURED_PIN_PROOF.md", + "visible": true, + "inside": true + } + }, + { + "width": 390, + "browser": "156.0.8078.47", + "initial": { + "scrollTop": 0, + "pageScroll": 397, + "heading": "math.uniform-matrix-cap-lifetime", + "headingTop": 292.046875, + "headingBottom": 343.890625, + "viewport": 844, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 390 + }, + "after_read": { + "scrollTop": 0, + "pageScroll": 1897, + "heading": "math.uniform-matrix-cap-lifetime", + "headingTop": -1207.953125, + "headingBottom": -1156.109375, + "viewport": 844, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 390 + }, + "new_selection": { + "scrollTop": 0, + "pageScroll": 397, + "heading": "math.d5-component.punctured-pin-proof", + "headingTop": 292.046875, + "headingBottom": 369.8125, + "viewport": 844, + "overflow": "visible", + "maxHeight": "none", + "documentWidth": 390 + }, + "selection_focus": "atlas-inspector", + "keyboard_source": { + "tag": "A", + "href": "https://github.com/d6g8k5htny-coder/Math-/blob/dbe6c0a1b1a22c7404b0fd173ee0043b98a08cde/reviews/d5_local_collar_20260928/PUNCTURED_PIN_PROOF.md", + "visible": true, + "inside": true + } + } +] diff --git a/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/optimized/result.json b/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/optimized/result.json new file mode 100644 index 000000000..0e4fc6273 --- /dev/null +++ b/docs/cbz/evidence/atlas-browser-20261029/original/atlas-regressions/optimized/result.json @@ -0,0 +1,45 @@ +{ + "errors": 1, + "expected_failures": 0, + "expected_ids": [ + "tests.test_atlas_focus_geometry.AtlasListFocusGeometry.test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary", + "tests.test_atlas_reader_scroll.AtlasReaderScroll.test_new_selection_after_a_long_read_keeps_its_heading_visible" + ], + "failures": 0, + "passed": false, + "runtime": { + "browsers": [ + { + "module": "tests.test_atlas_focus_geometry", + "path": "/usr/local/share/chromium/chrome-linux/chrome", + "sha256": "c97b58644d050f2affcb5132b7cb5690162ad459dfbbbaca1b3ef85cadcf2f21", + "version": "Chromium 156.0.8078.0" + }, + { + "module": "tests.test_atlas_reader_scroll", + "path": "/opt/google/chrome/google-chrome", + "sha256": "aea09d69ce7f24d5901f6bfb15dd44d0c856e793e0a498f8d8393ec7d2c308ec", + "version": "Google Chrome 156.0.8078.47" + } + ], + "optimize": 1, + "playwright": "1.62.0", + "python": "3.12.15 (main, Oct 1 2026, 02:35:59) [GCC 13.3.0]" + }, + "schema": "atlas-browser-regressions/1", + "scientific_effect": "NONE", + "skips": [], + "source_after": { + "commit": "6abed714e07816e74be07ec5cd3fd6f94e6d046e", + "tree": "535097c43edf631ede9b8ee21b1697b497b550c0" + }, + "source_before": { + "commit": "6abed714e07816e74be07ec5cd3fd6f94e6d046e", + "tree": "535097c43edf631ede9b8ee21b1697b497b550c0" + }, + "successful_ids": [ + "tests.test_atlas_reader_scroll.AtlasReaderScroll.test_new_selection_after_a_long_read_keeps_its_heading_visible" + ], + "tests_run": 2, + "unexpected_successes": 0 +} diff --git a/docs/cbz/evidence/browser-code-identity-20261029/README.md b/docs/cbz/evidence/browser-code-identity-20261029/README.md new file mode 100644 index 000000000..1cff7a1d8 --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/README.md @@ -0,0 +1,16 @@ +# Exact code-object identity repair + +Parent: `8d24c37ff52c012294d71bbcbf08bd24fae4e22a`. A newly added regression first produced an actual assertion +failure: `fail.__code__.replace(co_filename='subject_fake.py')` is a distinct +but structurally equal code object and was accepted by equality-based membership. +The RED log is retained. Tuple storage plus explicit identity comparisons now +refuse the clone. A grader mutation restores equality and is caught by the new +regression, graded through the pristine structured origin classifier. + +New observed population: 31 module /46 combined controls per mode; seven grader safeguard mutations. +Commands remain in the parent tool verification documentation; these tests do +not launch a browser or perform a full certificate replay. Pinned CPython3.11.16 +is used for final local runs. Original production/workflow/packet bytes and all +prior test methods are preserved. Previous evidence is retained at its original +source, not relabeled as current. No full repository, hosted, mathematical or +CBZ-0 certification is claimed. diff --git a/docs/cbz/evidence/browser-code-identity-20261029/SHA256SUMS.json b/docs/cbz/evidence/browser-code-identity-20261029/SHA256SUMS.json new file mode 100644 index 000000000..3aca4a579 --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/SHA256SUMS.json @@ -0,0 +1,11 @@ +{ + "README.md": "98b2b00935b847e126944630a95f34fa0b2854afa2050553b45626995eeae1ba", + "mutations-normal.json": "57cb7a862d89111ef676a2e060cd2aad99256512c8165f1ae3740506eff550fa", + "mutations-normal.stderr": "1dcc6bb9d77f7e6f02fc2f854fa351c6a54ad682e8f46f7a65ebe9ed2b1d0fc0", + "mutations-optimized.json": "6120a43f901b7eeefffc87e5c5153516e71a6117c13b15b3b1c3ef71ec813d5a", + "mutations-optimized.stderr": "1dcc6bb9d77f7e6f02fc2f854fa351c6a54ad682e8f46f7a65ebe9ed2b1d0fc0", + "normal.log": "9c4e5cb39ae0b15c461e0dd4833928573e0fbf5425140ac2309f2b4c36df4907", + "optimized.log": "abcea7a39ca72e1b811dadaadd4f549c3d20710f2d9647d1a64ef3f3d030283e", + "preservation.json": "1762f2bc570851b196f5d7a73178a778c8bf16f06de6c07e42a6b1c6b65729ff", + "red.log": "e337ad79ba81713bdfe005c90f22354fd5119544ec63c9b6b764fa3ad4cafdab" +} diff --git a/docs/cbz/evidence/browser-code-identity-20261029/mutations-normal.json b/docs/cbz/evidence/browser-code-identity-20261029/mutations-normal.json new file mode 100644 index 000000000..b53df583a --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/mutations-normal.json @@ -0,0 +1,55 @@ +{ + "optimized": 0, + "source_sha256": "08ca3dbdaf40e7a48caf658c431d4242c3ec95cf0657bf318357ea17fbf2ec30", + "results": [ + { + "name": "code_identity", + "method": "test_cloned_approved_code_is_not_authorized_identity", + "intended_assertion_kill": true, + "failures": 1, + "errors": 0 + }, + { + "name": "origin_frames", + "method": "test_subject_helper_callback_assertions_cannot_kill", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "phase", + "method": "test_later_phase_calling_selected_body_cannot_kill", + "intended_assertion_kill": true, + "failures": 1, + "errors": 0 + }, + { + "name": "origin_admission", + "method": "test_subject_helper_callback_assertions_cannot_kill", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "lazy_bodies", + "method": "test_deferred_bodies_refused", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "baseline", + "method": "test_error_skip_expected_failure_refused", + "intended_assertion_kill": true, + "failures": 4, + "errors": 0 + }, + { + "name": "terminal_accounting", + "method": "test_missing_or_duplicate_terminal_success_refused", + "intended_assertion_kill": true, + "failures": 2, + "errors": 0 + } + ] +} diff --git a/docs/cbz/evidence/browser-code-identity-20261029/mutations-normal.stderr b/docs/cbz/evidence/browser-code-identity-20261029/mutations-normal.stderr new file mode 100644 index 000000000..d47f219ea --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/mutations-normal.stderr @@ -0,0 +1,3 @@ +/workspace/shared/cbz-recovery/dd/runtime/python/lib/python3.11/unittest/case.py:623: RuntimeWarning: coroutine 'OracleOriginControls.test_deferred_bodies_refused..coroutine' was never awaited + self._callTestMethod(testMethod) +RuntimeWarning: Enable tracemalloc to get the object allocation traceback diff --git a/docs/cbz/evidence/browser-code-identity-20261029/mutations-optimized.json b/docs/cbz/evidence/browser-code-identity-20261029/mutations-optimized.json new file mode 100644 index 000000000..b57572997 --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/mutations-optimized.json @@ -0,0 +1,55 @@ +{ + "optimized": 1, + "source_sha256": "08ca3dbdaf40e7a48caf658c431d4242c3ec95cf0657bf318357ea17fbf2ec30", + "results": [ + { + "name": "code_identity", + "method": "test_cloned_approved_code_is_not_authorized_identity", + "intended_assertion_kill": true, + "failures": 1, + "errors": 0 + }, + { + "name": "origin_frames", + "method": "test_subject_helper_callback_assertions_cannot_kill", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "phase", + "method": "test_later_phase_calling_selected_body_cannot_kill", + "intended_assertion_kill": true, + "failures": 1, + "errors": 0 + }, + { + "name": "origin_admission", + "method": "test_subject_helper_callback_assertions_cannot_kill", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "lazy_bodies", + "method": "test_deferred_bodies_refused", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "baseline", + "method": "test_error_skip_expected_failure_refused", + "intended_assertion_kill": true, + "failures": 4, + "errors": 0 + }, + { + "name": "terminal_accounting", + "method": "test_missing_or_duplicate_terminal_success_refused", + "intended_assertion_kill": true, + "failures": 2, + "errors": 0 + } + ] +} diff --git a/docs/cbz/evidence/browser-code-identity-20261029/mutations-optimized.stderr b/docs/cbz/evidence/browser-code-identity-20261029/mutations-optimized.stderr new file mode 100644 index 000000000..d47f219ea --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/mutations-optimized.stderr @@ -0,0 +1,3 @@ +/workspace/shared/cbz-recovery/dd/runtime/python/lib/python3.11/unittest/case.py:623: RuntimeWarning: coroutine 'OracleOriginControls.test_deferred_bodies_refused..coroutine' was never awaited + self._callTestMethod(testMethod) +RuntimeWarning: Enable tracemalloc to get the object allocation traceback diff --git a/docs/cbz/evidence/browser-code-identity-20261029/normal.log b/docs/cbz/evidence/browser-code-identity-20261029/normal.log new file mode 100644 index 000000000..e20a971ca --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/normal.log @@ -0,0 +1,51 @@ +test_both_tests_select_override_then_chrome_then_chromium (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_both_tests_select_override_then_chrome_then_chromium) ... ok +test_duplicate_inventory_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_duplicate_inventory_refused) ... ok +test_empty_inventory_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_empty_inventory_refused) ... ok +test_exact_two_successes_pass (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_exact_two_successes_pass) ... ok +test_expected_failure_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_expected_failure_is_not_success) ... ok +test_failures_errors_and_subtests_are_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_failures_errors_and_subtests_are_not_success) ... ok +test_focus_and_reader_explicitly_enable_sandbox (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_focus_and_reader_explicitly_enable_sandbox) ... ok +test_main_success_and_failure_exit_match_new_report (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_main_success_and_failure_exit_match_new_report) ... ok +test_missing_execution_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_missing_execution_is_not_success) ... ok +test_reused_output_is_refused_without_overwrite (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_reused_output_is_refused_without_overwrite) ... ok +test_runtime_identity_records_each_test_selected_launcher (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_runtime_identity_records_each_test_selected_launcher) ... ok +test_six_runner_source_mutants_are_killed_by_assertions (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_six_runner_source_mutants_are_killed_by_assertions) ... ok +test_skip_is_failure (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_skip_is_failure) ... ok +test_source_drift_cannot_leave_successful_report (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_source_drift_cannot_leave_successful_report) ... ok +test_unexpected_success_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_unexpected_success_is_not_success) ... ok +test_workflow_runs_both_modes_and_retains_evidence (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_workflow_runs_both_modes_and_retains_evidence) ... ok +test_wrong_and_extra_ids_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_wrong_and_extra_ids_refused) ... ok +test_cleanup_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_cleanup_assertions_cannot_kill) ... ok +test_cloned_approved_code_is_not_authorized_identity (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_cloned_approved_code_is_not_authorized_identity) ... ok +test_deferred_bodies_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_deferred_bodies_refused) ... ok +test_empty_duplicate_population_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_empty_duplicate_population_refused) ... ok +test_error_skip_expected_failure_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_error_skip_expected_failure_refused) ... ok +test_fixture_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_fixture_assertions_cannot_kill) ... ok +test_forged_unittest_filename_is_not_authority (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_forged_unittest_filename_is_not_authority) ... ok +test_genuine_baseline_and_assertion_origins (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_genuine_baseline_and_assertion_origins) ... ok +test_later_phase_calling_selected_body_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_later_phase_calling_selected_body_cannot_kill) ... ok +test_missing_or_duplicate_terminal_success_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_missing_or_duplicate_terminal_success_refused) ... ok +test_repeated_body_execution_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_repeated_body_execution_refused) ... ok +test_setup_calling_body_does_not_count_as_execution (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_setup_calling_body_does_not_count_as_execution) ... ok +test_subject_helper_callback_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_subject_helper_callback_assertions_cannot_kill) ... ok +test_synchronous_returning_deferred_or_value_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_synchronous_returning_deferred_or_value_refused) ... ok +test_missing_and_malformed_source_write_failure_report (tests.test_current_site_browser_contract.BrowserReportFailureTests.test_missing_and_malformed_source_write_failure_report) ... ok +test_published_overlay_and_snapshot_only_static_reading_have_separate_exact_expectations (tests.test_current_site_browser_contract.ResumeContractTests.test_published_overlay_and_snapshot_only_static_reading_have_separate_exact_expectations) ... ok +test_wrong_shared_fields_phantom_links_and_false_operational_state_are_refused (tests.test_current_site_browser_contract.ResumeContractTests.test_wrong_shared_fields_phantom_links_and_false_operational_state_are_refused) ... ok +test_browser_dependencies_are_separate_and_version_pinned (tests.test_public_shop_browser_contract.BrowserCheckContract.test_browser_dependencies_are_separate_and_version_pinned) ... ok +test_browser_job_is_read_only_and_retains_failure_artifacts (tests.test_public_shop_browser_contract.BrowserCheckContract.test_browser_job_is_read_only_and_retains_failure_artifacts) ... ok +test_runner_is_loopback_only_with_a_four_case_matrix (tests.test_public_shop_browser_contract.BrowserCheckContract.test_runner_is_loopback_only_with_a_four_case_matrix) ... ok +test_bibtex_browser_checks_compare_real_clipboard_and_keep_failure_paths (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_bibtex_browser_checks_compare_real_clipboard_and_keep_failure_paths) ... ok +test_bibtex_disabled_checks_keep_the_hidden_button_addressable (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_bibtex_disabled_checks_keep_the_hidden_button_addressable) ... ok +test_clipboard_denial_fixture_reinitializes_after_same_document_history (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_clipboard_denial_fixture_reinitializes_after_same_document_history) ... ok +test_existing_runner_covers_source_quote_and_keyboard_disclosure (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_existing_runner_covers_source_quote_and_keyboard_disclosure) ... ok +test_new_reader_controls_are_reached_by_the_existing_browser_flows (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_new_reader_controls_are_reached_by_the_existing_browser_flows) ... ok +test_other_teaching_exports_are_reached_in_the_four_source_cases (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_other_teaching_exports_are_reached_in_the_four_source_cases) ... ok +test_pin_source_enumeration_preserves_the_downloaded_artifact_path (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_pin_source_enumeration_preserves_the_downloaded_artifact_path) ... ok +test_reader_tools_have_desktop_and_mobile_browser_evidence (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_reader_tools_have_desktop_and_mobile_browser_evidence) ... ok +test_rendered_layout_cases_read_what_the_browser_drew (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_rendered_layout_cases_read_what_the_browser_drew) ... ok + +---------------------------------------------------------------------- +Ran 46 tests in 0.910s + +OK diff --git a/docs/cbz/evidence/browser-code-identity-20261029/optimized.log b/docs/cbz/evidence/browser-code-identity-20261029/optimized.log new file mode 100644 index 000000000..883331784 --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/optimized.log @@ -0,0 +1,51 @@ +test_both_tests_select_override_then_chrome_then_chromium (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_both_tests_select_override_then_chrome_then_chromium) ... ok +test_duplicate_inventory_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_duplicate_inventory_refused) ... ok +test_empty_inventory_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_empty_inventory_refused) ... ok +test_exact_two_successes_pass (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_exact_two_successes_pass) ... ok +test_expected_failure_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_expected_failure_is_not_success) ... ok +test_failures_errors_and_subtests_are_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_failures_errors_and_subtests_are_not_success) ... ok +test_focus_and_reader_explicitly_enable_sandbox (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_focus_and_reader_explicitly_enable_sandbox) ... ok +test_main_success_and_failure_exit_match_new_report (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_main_success_and_failure_exit_match_new_report) ... ok +test_missing_execution_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_missing_execution_is_not_success) ... ok +test_reused_output_is_refused_without_overwrite (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_reused_output_is_refused_without_overwrite) ... ok +test_runtime_identity_records_each_test_selected_launcher (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_runtime_identity_records_each_test_selected_launcher) ... ok +test_six_runner_source_mutants_are_killed_by_assertions (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_six_runner_source_mutants_are_killed_by_assertions) ... ok +test_skip_is_failure (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_skip_is_failure) ... ok +test_source_drift_cannot_leave_successful_report (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_source_drift_cannot_leave_successful_report) ... ok +test_unexpected_success_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_unexpected_success_is_not_success) ... ok +test_workflow_runs_both_modes_and_retains_evidence (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_workflow_runs_both_modes_and_retains_evidence) ... ok +test_wrong_and_extra_ids_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_wrong_and_extra_ids_refused) ... ok +test_cleanup_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_cleanup_assertions_cannot_kill) ... ok +test_cloned_approved_code_is_not_authorized_identity (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_cloned_approved_code_is_not_authorized_identity) ... ok +test_deferred_bodies_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_deferred_bodies_refused) ... ok +test_empty_duplicate_population_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_empty_duplicate_population_refused) ... ok +test_error_skip_expected_failure_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_error_skip_expected_failure_refused) ... ok +test_fixture_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_fixture_assertions_cannot_kill) ... ok +test_forged_unittest_filename_is_not_authority (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_forged_unittest_filename_is_not_authority) ... ok +test_genuine_baseline_and_assertion_origins (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_genuine_baseline_and_assertion_origins) ... ok +test_later_phase_calling_selected_body_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_later_phase_calling_selected_body_cannot_kill) ... ok +test_missing_or_duplicate_terminal_success_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_missing_or_duplicate_terminal_success_refused) ... ok +test_repeated_body_execution_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_repeated_body_execution_refused) ... ok +test_setup_calling_body_does_not_count_as_execution (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_setup_calling_body_does_not_count_as_execution) ... ok +test_subject_helper_callback_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_subject_helper_callback_assertions_cannot_kill) ... ok +test_synchronous_returning_deferred_or_value_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_synchronous_returning_deferred_or_value_refused) ... ok +test_missing_and_malformed_source_write_failure_report (tests.test_current_site_browser_contract.BrowserReportFailureTests.test_missing_and_malformed_source_write_failure_report) ... ok +test_published_overlay_and_snapshot_only_static_reading_have_separate_exact_expectations (tests.test_current_site_browser_contract.ResumeContractTests.test_published_overlay_and_snapshot_only_static_reading_have_separate_exact_expectations) ... ok +test_wrong_shared_fields_phantom_links_and_false_operational_state_are_refused (tests.test_current_site_browser_contract.ResumeContractTests.test_wrong_shared_fields_phantom_links_and_false_operational_state_are_refused) ... ok +test_browser_dependencies_are_separate_and_version_pinned (tests.test_public_shop_browser_contract.BrowserCheckContract.test_browser_dependencies_are_separate_and_version_pinned) ... ok +test_browser_job_is_read_only_and_retains_failure_artifacts (tests.test_public_shop_browser_contract.BrowserCheckContract.test_browser_job_is_read_only_and_retains_failure_artifacts) ... ok +test_runner_is_loopback_only_with_a_four_case_matrix (tests.test_public_shop_browser_contract.BrowserCheckContract.test_runner_is_loopback_only_with_a_four_case_matrix) ... ok +test_bibtex_browser_checks_compare_real_clipboard_and_keep_failure_paths (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_bibtex_browser_checks_compare_real_clipboard_and_keep_failure_paths) ... ok +test_bibtex_disabled_checks_keep_the_hidden_button_addressable (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_bibtex_disabled_checks_keep_the_hidden_button_addressable) ... ok +test_clipboard_denial_fixture_reinitializes_after_same_document_history (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_clipboard_denial_fixture_reinitializes_after_same_document_history) ... ok +test_existing_runner_covers_source_quote_and_keyboard_disclosure (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_existing_runner_covers_source_quote_and_keyboard_disclosure) ... ok +test_new_reader_controls_are_reached_by_the_existing_browser_flows (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_new_reader_controls_are_reached_by_the_existing_browser_flows) ... ok +test_other_teaching_exports_are_reached_in_the_four_source_cases (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_other_teaching_exports_are_reached_in_the_four_source_cases) ... ok +test_pin_source_enumeration_preserves_the_downloaded_artifact_path (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_pin_source_enumeration_preserves_the_downloaded_artifact_path) ... ok +test_reader_tools_have_desktop_and_mobile_browser_evidence (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_reader_tools_have_desktop_and_mobile_browser_evidence) ... ok +test_rendered_layout_cases_read_what_the_browser_drew (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_rendered_layout_cases_read_what_the_browser_drew) ... ok + +---------------------------------------------------------------------- +Ran 46 tests in 1.068s + +OK diff --git a/docs/cbz/evidence/browser-code-identity-20261029/preservation.json b/docs/cbz/evidence/browser-code-identity-20261029/preservation.json new file mode 100644 index 000000000..66725f572 --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/preservation.json @@ -0,0 +1,15 @@ +{ + "parent": "8d24c37ff52c012294d71bbcbf08bd24fae4e22a", + "source_sha256": "08ca3dbdaf40e7a48caf658c431d4242c3ec95cf0657bf318357ea17fbf2ec30", + "all_original_named_tests_retained": true, + "modified_original_methods": [], + "protected_paths_unchanged": [ + "tools/run_atlas_browser_regressions.py", + "tests/test_atlas_focus_geometry.py", + "tests/test_atlas_reader_scroll.py", + ".github/workflows/public-shop.yml" + ], + "new_test_methods": [ + "OracleOriginControls.test_cloned_approved_code_is_not_authorized_identity" + ] +} diff --git a/docs/cbz/evidence/browser-code-identity-20261029/red.log b/docs/cbz/evidence/browser-code-identity-20261029/red.log new file mode 100644 index 000000000..2c9334c1f --- /dev/null +++ b/docs/cbz/evidence/browser-code-identity-20261029/red.log @@ -0,0 +1,14 @@ +test_cloned_approved_code_is_not_authorized_identity (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_cloned_approved_code_is_not_authorized_identity) ... FAIL + +====================================================================== +FAIL: test_cloned_approved_code_is_not_authorized_identity (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_cloned_approved_code_is_not_authorized_identity) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/workspace/shared/cbz-recovery/oracle-audit/browser/tests/test_atlas_browser_regressions_runner.py", line 400, in test_cloned_approved_code_is_not_authorized_identity + self.assertFalse(self.observe(body).oracle_killed) +AssertionError: True is not false + +---------------------------------------------------------------------- +Ran 1 test in 0.001s + +FAILED (failures=1) diff --git a/docs/cbz/evidence/browser-origin-20261029/README.md b/docs/cbz/evidence/browser-origin-20261029/README.md new file mode 100644 index 000000000..ccfb972a7 --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/README.md @@ -0,0 +1,24 @@ +# Browser mutation-grader origin correction + +Scoped engineering evidence, 2026-10-29. Original false acceptance was reproduced +against source tree6786ec405cef50e0fef73c7fcd9ca3945c555230. Monkeypatching only +the child class setUp to raise AssertionError made the old six-mutant meta-test +accept every failed setup as a kill. The actual test bodies never ran. Both +normal and optimized observations are retained. DD lines in those original +combined logs refer to the separately repaired Math541 harness, not browser scope. + +Final local commands and exact limits are documented in +`docs/cbz/ATLAS_BROWSER_REGRESSIONS.md`. All45 combined browser-contract controls +passed in both modes, including30 controls in the amended module, original six +production source mutants and13 new lifecycle/origin controls. Six additional +grader guard mutations per mode were graded with a pristine independent copy +of the repaired structured classifier, not the mutant's own admission flags. +Raw stdout/stderr are preserved; warning in the lazy-body adverse mutant is not +a passing oracle. Full repository and actual browser execution on this source +remain unexecuted here. Production runner/browser/workflow files are unchanged. + +Parent review independently reran30methods normally and optimized, checked the +raw-origin classifier and pristine meta-runner, and required explicit target +cardinality refusal under -O and portable repository-relative paths. Both were +corrected before final rerun. This is same-family engineering review, not a +formal PR approval, mathematical proof, universal harness soundness or CBZ-0. diff --git a/docs/cbz/evidence/browser-origin-20261029/check_grader_mutations.py b/docs/cbz/evidence/browser-origin-20261029/check_grader_mutations.py new file mode 100644 index 000000000..e48c6ba70 --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/check_grader_mutations.py @@ -0,0 +1,28 @@ +import importlib.util,sys,json,tempfile +from pathlib import Path +ROOT=Path(__file__).resolve().parents[4] +PATH=ROOT/'tests/test_atlas_browser_regressions_runner.py' +def load(name,path): + spec=importlib.util.spec_from_file_location(name,path);m=importlib.util.module_from_spec(spec);sys.modules[name]=m;spec.loader.exec_module(m);return m +trusted=load('trusted_browser_origin',PATH);source=PATH.read_text();rows=[] +cases=[ + ('code_identity','any(code is allowed for allowed in _ASSERTION_CODES)','code in _ASSERTION_CODES','test_cloned_approved_code_is_not_authorized_identity'), + ('origin_frames','and all(any(code is allowed for allowed in _ASSERTION_CODES) for code in codes[positions[0]+1:])','and True','test_subject_helper_callback_assertions_cannot_kill'), + ('phase',"and state.get('phase') in ('body', 'body-exited')",'and True','test_later_phase_calling_selected_body_cannot_kill'), + ('origin_admission','and all(v for _, v in result.origins)','and True','test_subject_helper_callback_assertions_cannot_kill'), + ('lazy_bodies',' or inspect.iscoroutinefunction(selected) or inspect.isgeneratorfunction(selected) or inspect.isasyncgenfunction(selected)','', 'test_deferred_bodies_refused'), + ('baseline','result.oracle_baseline = complete and result.wasSuccessful() and not result.origins','result.oracle_baseline = True','test_error_skip_expected_failure_refused'), + ('terminal_accounting','complete = (complete and accounted == set(expected)\n and len(result.successful) == len(set(result.successful))\n and not (set(result.successful) & {identity for identity, unused in result.origins}))','complete = complete','test_missing_or_duplicate_terminal_success_refused'), +] +for name,before,after,method in cases: + if source.count(before) != 1: raise RuntimeError('nonunique mutation target: '+name) + baseline=trusted.run_oracle_controls(trusted.OracleOriginControls,(method,)) + if not baseline.oracle_baseline:raise RuntimeError('bad baseline '+name) + with tempfile.TemporaryDirectory() as tmp: + p=Path(tmp)/'mutant.py';changed=source.replace(before,after,1) + if name=='lazy_bodies': changed=changed.replace('if returned is not None:', 'if False:') + p.write_text(changed);mutant=load('browser_origin_'+name,p) + result=trusted.run_oracle_controls(mutant.OracleOriginControls,(method,)) + if not result.oracle_killed:raise RuntimeError('not intended assertion kill '+name+repr(result.failures)+repr(result.errors)) + rows.append({'name':name,'method':method,'intended_assertion_kill':True,'failures':len(result.failures),'errors':len(result.errors)}) +print(json.dumps({'optimized':sys.flags.optimize,'source_sha256':__import__('hashlib').sha256(source.encode()).hexdigest(),'results':rows},indent=2)) diff --git a/docs/cbz/evidence/browser-origin-20261029/mutations-normal.json b/docs/cbz/evidence/browser-origin-20261029/mutations-normal.json new file mode 100644 index 000000000..64f27983f --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/mutations-normal.json @@ -0,0 +1,48 @@ +{ + "optimized": 0, + "source_sha256": "2d2b2d477f632cf86096d623cf6b29616c52e3cf89328b280f995acdeeefb0c6", + "results": [ + { + "name": "origin_frames", + "method": "test_subject_helper_callback_assertions_cannot_kill", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "phase", + "method": "test_later_phase_calling_selected_body_cannot_kill", + "intended_assertion_kill": true, + "failures": 1, + "errors": 0 + }, + { + "name": "origin_admission", + "method": "test_subject_helper_callback_assertions_cannot_kill", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "lazy_bodies", + "method": "test_deferred_bodies_refused", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "baseline", + "method": "test_error_skip_expected_failure_refused", + "intended_assertion_kill": true, + "failures": 4, + "errors": 0 + }, + { + "name": "terminal_accounting", + "method": "test_missing_or_duplicate_terminal_success_refused", + "intended_assertion_kill": true, + "failures": 2, + "errors": 0 + } + ] +} diff --git a/docs/cbz/evidence/browser-origin-20261029/mutations-normal.stderr b/docs/cbz/evidence/browser-origin-20261029/mutations-normal.stderr new file mode 100644 index 000000000..d099706a2 --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/mutations-normal.stderr @@ -0,0 +1,3 @@ +/opt/codex/runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/unittest/case.py:634: RuntimeWarning: coroutine 'OracleOriginControls.test_deferred_bodies_refused..coroutine' was never awaited + self._callTestMethod(testMethod) +RuntimeWarning: Enable tracemalloc to get the object allocation traceback diff --git a/docs/cbz/evidence/browser-origin-20261029/mutations-optimized.json b/docs/cbz/evidence/browser-origin-20261029/mutations-optimized.json new file mode 100644 index 000000000..c51a0e2bd --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/mutations-optimized.json @@ -0,0 +1,48 @@ +{ + "optimized": 1, + "source_sha256": "2d2b2d477f632cf86096d623cf6b29616c52e3cf89328b280f995acdeeefb0c6", + "results": [ + { + "name": "origin_frames", + "method": "test_subject_helper_callback_assertions_cannot_kill", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "phase", + "method": "test_later_phase_calling_selected_body_cannot_kill", + "intended_assertion_kill": true, + "failures": 1, + "errors": 0 + }, + { + "name": "origin_admission", + "method": "test_subject_helper_callback_assertions_cannot_kill", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "lazy_bodies", + "method": "test_deferred_bodies_refused", + "intended_assertion_kill": true, + "failures": 3, + "errors": 0 + }, + { + "name": "baseline", + "method": "test_error_skip_expected_failure_refused", + "intended_assertion_kill": true, + "failures": 4, + "errors": 0 + }, + { + "name": "terminal_accounting", + "method": "test_missing_or_duplicate_terminal_success_refused", + "intended_assertion_kill": true, + "failures": 2, + "errors": 0 + } + ] +} diff --git a/docs/cbz/evidence/browser-origin-20261029/mutations-optimized.stderr b/docs/cbz/evidence/browser-origin-20261029/mutations-optimized.stderr new file mode 100644 index 000000000..d099706a2 --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/mutations-optimized.stderr @@ -0,0 +1,3 @@ +/opt/codex/runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/unittest/case.py:634: RuntimeWarning: coroutine 'OracleOriginControls.test_deferred_bodies_refused..coroutine' was never awaited + self._callTestMethod(testMethod) +RuntimeWarning: Enable tracemalloc to get the object allocation traceback diff --git a/docs/cbz/evidence/browser-origin-20261029/normal.log b/docs/cbz/evidence/browser-origin-20261029/normal.log new file mode 100644 index 000000000..6e225ad0f --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/normal.log @@ -0,0 +1,50 @@ +test_both_tests_select_override_then_chrome_then_chromium (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_both_tests_select_override_then_chrome_then_chromium) ... ok +test_duplicate_inventory_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_duplicate_inventory_refused) ... ok +test_empty_inventory_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_empty_inventory_refused) ... ok +test_exact_two_successes_pass (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_exact_two_successes_pass) ... ok +test_expected_failure_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_expected_failure_is_not_success) ... ok +test_failures_errors_and_subtests_are_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_failures_errors_and_subtests_are_not_success) ... ok +test_focus_and_reader_explicitly_enable_sandbox (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_focus_and_reader_explicitly_enable_sandbox) ... ok +test_main_success_and_failure_exit_match_new_report (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_main_success_and_failure_exit_match_new_report) ... ok +test_missing_execution_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_missing_execution_is_not_success) ... ok +test_reused_output_is_refused_without_overwrite (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_reused_output_is_refused_without_overwrite) ... ok +test_runtime_identity_records_each_test_selected_launcher (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_runtime_identity_records_each_test_selected_launcher) ... ok +test_six_runner_source_mutants_are_killed_by_assertions (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_six_runner_source_mutants_are_killed_by_assertions) ... ok +test_skip_is_failure (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_skip_is_failure) ... ok +test_source_drift_cannot_leave_successful_report (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_source_drift_cannot_leave_successful_report) ... ok +test_unexpected_success_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_unexpected_success_is_not_success) ... ok +test_workflow_runs_both_modes_and_retains_evidence (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_workflow_runs_both_modes_and_retains_evidence) ... ok +test_wrong_and_extra_ids_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_wrong_and_extra_ids_refused) ... ok +test_cleanup_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_cleanup_assertions_cannot_kill) ... ok +test_deferred_bodies_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_deferred_bodies_refused) ... ok +test_empty_duplicate_population_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_empty_duplicate_population_refused) ... ok +test_error_skip_expected_failure_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_error_skip_expected_failure_refused) ... ok +test_fixture_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_fixture_assertions_cannot_kill) ... ok +test_forged_unittest_filename_is_not_authority (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_forged_unittest_filename_is_not_authority) ... ok +test_genuine_baseline_and_assertion_origins (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_genuine_baseline_and_assertion_origins) ... ok +test_later_phase_calling_selected_body_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_later_phase_calling_selected_body_cannot_kill) ... ok +test_missing_or_duplicate_terminal_success_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_missing_or_duplicate_terminal_success_refused) ... ok +test_repeated_body_execution_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_repeated_body_execution_refused) ... ok +test_setup_calling_body_does_not_count_as_execution (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_setup_calling_body_does_not_count_as_execution) ... ok +test_subject_helper_callback_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_subject_helper_callback_assertions_cannot_kill) ... ok +test_synchronous_returning_deferred_or_value_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_synchronous_returning_deferred_or_value_refused) ... ok +test_missing_and_malformed_source_write_failure_report (tests.test_current_site_browser_contract.BrowserReportFailureTests.test_missing_and_malformed_source_write_failure_report) ... ok +test_published_overlay_and_snapshot_only_static_reading_have_separate_exact_expectations (tests.test_current_site_browser_contract.ResumeContractTests.test_published_overlay_and_snapshot_only_static_reading_have_separate_exact_expectations) ... ok +test_wrong_shared_fields_phantom_links_and_false_operational_state_are_refused (tests.test_current_site_browser_contract.ResumeContractTests.test_wrong_shared_fields_phantom_links_and_false_operational_state_are_refused) ... ok +test_browser_dependencies_are_separate_and_version_pinned (tests.test_public_shop_browser_contract.BrowserCheckContract.test_browser_dependencies_are_separate_and_version_pinned) ... ok +test_browser_job_is_read_only_and_retains_failure_artifacts (tests.test_public_shop_browser_contract.BrowserCheckContract.test_browser_job_is_read_only_and_retains_failure_artifacts) ... ok +test_runner_is_loopback_only_with_a_four_case_matrix (tests.test_public_shop_browser_contract.BrowserCheckContract.test_runner_is_loopback_only_with_a_four_case_matrix) ... ok +test_bibtex_browser_checks_compare_real_clipboard_and_keep_failure_paths (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_bibtex_browser_checks_compare_real_clipboard_and_keep_failure_paths) ... ok +test_bibtex_disabled_checks_keep_the_hidden_button_addressable (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_bibtex_disabled_checks_keep_the_hidden_button_addressable) ... ok +test_clipboard_denial_fixture_reinitializes_after_same_document_history (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_clipboard_denial_fixture_reinitializes_after_same_document_history) ... ok +test_existing_runner_covers_source_quote_and_keyboard_disclosure (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_existing_runner_covers_source_quote_and_keyboard_disclosure) ... ok +test_new_reader_controls_are_reached_by_the_existing_browser_flows (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_new_reader_controls_are_reached_by_the_existing_browser_flows) ... ok +test_other_teaching_exports_are_reached_in_the_four_source_cases (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_other_teaching_exports_are_reached_in_the_four_source_cases) ... ok +test_pin_source_enumeration_preserves_the_downloaded_artifact_path (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_pin_source_enumeration_preserves_the_downloaded_artifact_path) ... ok +test_reader_tools_have_desktop_and_mobile_browser_evidence (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_reader_tools_have_desktop_and_mobile_browser_evidence) ... ok +test_rendered_layout_cases_read_what_the_browser_drew (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_rendered_layout_cases_read_what_the_browser_drew) ... ok + +---------------------------------------------------------------------- +Ran 45 tests in 0.611s + +OK diff --git a/docs/cbz/evidence/browser-origin-20261029/optimized.log b/docs/cbz/evidence/browser-origin-20261029/optimized.log new file mode 100644 index 000000000..363372752 --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/optimized.log @@ -0,0 +1,50 @@ +test_both_tests_select_override_then_chrome_then_chromium (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_both_tests_select_override_then_chrome_then_chromium) ... ok +test_duplicate_inventory_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_duplicate_inventory_refused) ... ok +test_empty_inventory_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_empty_inventory_refused) ... ok +test_exact_two_successes_pass (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_exact_two_successes_pass) ... ok +test_expected_failure_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_expected_failure_is_not_success) ... ok +test_failures_errors_and_subtests_are_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_failures_errors_and_subtests_are_not_success) ... ok +test_focus_and_reader_explicitly_enable_sandbox (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_focus_and_reader_explicitly_enable_sandbox) ... ok +test_main_success_and_failure_exit_match_new_report (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_main_success_and_failure_exit_match_new_report) ... ok +test_missing_execution_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_missing_execution_is_not_success) ... ok +test_reused_output_is_refused_without_overwrite (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_reused_output_is_refused_without_overwrite) ... ok +test_runtime_identity_records_each_test_selected_launcher (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_runtime_identity_records_each_test_selected_launcher) ... ok +test_six_runner_source_mutants_are_killed_by_assertions (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_six_runner_source_mutants_are_killed_by_assertions) ... ok +test_skip_is_failure (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_skip_is_failure) ... ok +test_source_drift_cannot_leave_successful_report (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_source_drift_cannot_leave_successful_report) ... ok +test_unexpected_success_is_not_success (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_unexpected_success_is_not_success) ... ok +test_workflow_runs_both_modes_and_retains_evidence (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_workflow_runs_both_modes_and_retains_evidence) ... ok +test_wrong_and_extra_ids_refused (tests.test_atlas_browser_regressions_runner.AtlasBrowserRunnerControls.test_wrong_and_extra_ids_refused) ... ok +test_cleanup_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_cleanup_assertions_cannot_kill) ... ok +test_deferred_bodies_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_deferred_bodies_refused) ... ok +test_empty_duplicate_population_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_empty_duplicate_population_refused) ... ok +test_error_skip_expected_failure_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_error_skip_expected_failure_refused) ... ok +test_fixture_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_fixture_assertions_cannot_kill) ... ok +test_forged_unittest_filename_is_not_authority (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_forged_unittest_filename_is_not_authority) ... ok +test_genuine_baseline_and_assertion_origins (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_genuine_baseline_and_assertion_origins) ... ok +test_later_phase_calling_selected_body_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_later_phase_calling_selected_body_cannot_kill) ... ok +test_missing_or_duplicate_terminal_success_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_missing_or_duplicate_terminal_success_refused) ... ok +test_repeated_body_execution_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_repeated_body_execution_refused) ... ok +test_setup_calling_body_does_not_count_as_execution (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_setup_calling_body_does_not_count_as_execution) ... ok +test_subject_helper_callback_assertions_cannot_kill (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_subject_helper_callback_assertions_cannot_kill) ... ok +test_synchronous_returning_deferred_or_value_refused (tests.test_atlas_browser_regressions_runner.OracleOriginControls.test_synchronous_returning_deferred_or_value_refused) ... ok +test_missing_and_malformed_source_write_failure_report (tests.test_current_site_browser_contract.BrowserReportFailureTests.test_missing_and_malformed_source_write_failure_report) ... ok +test_published_overlay_and_snapshot_only_static_reading_have_separate_exact_expectations (tests.test_current_site_browser_contract.ResumeContractTests.test_published_overlay_and_snapshot_only_static_reading_have_separate_exact_expectations) ... ok +test_wrong_shared_fields_phantom_links_and_false_operational_state_are_refused (tests.test_current_site_browser_contract.ResumeContractTests.test_wrong_shared_fields_phantom_links_and_false_operational_state_are_refused) ... ok +test_browser_dependencies_are_separate_and_version_pinned (tests.test_public_shop_browser_contract.BrowserCheckContract.test_browser_dependencies_are_separate_and_version_pinned) ... ok +test_browser_job_is_read_only_and_retains_failure_artifacts (tests.test_public_shop_browser_contract.BrowserCheckContract.test_browser_job_is_read_only_and_retains_failure_artifacts) ... ok +test_runner_is_loopback_only_with_a_four_case_matrix (tests.test_public_shop_browser_contract.BrowserCheckContract.test_runner_is_loopback_only_with_a_four_case_matrix) ... ok +test_bibtex_browser_checks_compare_real_clipboard_and_keep_failure_paths (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_bibtex_browser_checks_compare_real_clipboard_and_keep_failure_paths) ... ok +test_bibtex_disabled_checks_keep_the_hidden_button_addressable (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_bibtex_disabled_checks_keep_the_hidden_button_addressable) ... ok +test_clipboard_denial_fixture_reinitializes_after_same_document_history (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_clipboard_denial_fixture_reinitializes_after_same_document_history) ... ok +test_existing_runner_covers_source_quote_and_keyboard_disclosure (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_existing_runner_covers_source_quote_and_keyboard_disclosure) ... ok +test_new_reader_controls_are_reached_by_the_existing_browser_flows (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_new_reader_controls_are_reached_by_the_existing_browser_flows) ... ok +test_other_teaching_exports_are_reached_in_the_four_source_cases (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_other_teaching_exports_are_reached_in_the_four_source_cases) ... ok +test_pin_source_enumeration_preserves_the_downloaded_artifact_path (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_pin_source_enumeration_preserves_the_downloaded_artifact_path) ... ok +test_reader_tools_have_desktop_and_mobile_browser_evidence (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_reader_tools_have_desktop_and_mobile_browser_evidence) ... ok +test_rendered_layout_cases_read_what_the_browser_drew (tests.test_public_shop_browser_contract.ReaderSourceBrowserContract.test_rendered_layout_cases_read_what_the_browser_drew) ... ok + +---------------------------------------------------------------------- +Ran 45 tests in 0.690s + +OK diff --git a/docs/cbz/evidence/browser-origin-20261029/original-false-accept-normal.log b/docs/cbz/evidence/browser-origin-20261029/original-false-accept-normal.log new file mode 100644 index 000000000..da9318d13 --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/original-false-accept-normal.log @@ -0,0 +1,2 @@ +BROWSER_FALSE_ACCEPT: six setup-only failures accepted as mutant kills +DD_FALSE_ACCEPT: subject-origin AssertionError accepted as mutant kill diff --git a/docs/cbz/evidence/browser-origin-20261029/original-false-accept-optimized.log b/docs/cbz/evidence/browser-origin-20261029/original-false-accept-optimized.log new file mode 100644 index 000000000..da9318d13 --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/original-false-accept-optimized.log @@ -0,0 +1,2 @@ +BROWSER_FALSE_ACCEPT: six setup-only failures accepted as mutant kills +DD_FALSE_ACCEPT: subject-origin AssertionError accepted as mutant kill diff --git a/docs/cbz/evidence/browser-origin-20261029/preservation.json b/docs/cbz/evidence/browser-origin-20261029/preservation.json new file mode 100644 index 000000000..633520648 --- /dev/null +++ b/docs/cbz/evidence/browser-origin-20261029/preservation.json @@ -0,0 +1,15 @@ +{ + "base": "1390112fe58093d906c878043d6b985067610002", + "original_test_methods": 17, + "retained_method_names": true, + "changed_original_methods": [ + "test_six_runner_source_mutants_are_killed_by_assertions" + ], + "protected_paths_unchanged": [ + "tools/run_atlas_browser_regressions.py", + "tests/test_atlas_focus_geometry.py", + "tests/test_atlas_reader_scroll.py", + ".github/workflows/public-shop.yml" + ], + "test_source_sha256": "2d2b2d477f632cf86096d623cf6b29616c52e3cf89328b280f995acdeeefb0c6" +} diff --git a/tests/test_atlas_browser_regressions_runner.py b/tests/test_atlas_browser_regressions_runner.py new file mode 100644 index 000000000..d27833408 --- /dev/null +++ b/tests/test_atlas_browser_regressions_runner.py @@ -0,0 +1,476 @@ +"""Controls for exact two-test execution; these never launch a browser.""" +import ast +import importlib.util +import io +import inspect +import json +import tempfile +import types +import hashlib +from unittest import mock +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[1] +PATH = ROOT / 'tools/run_atlas_browser_regressions.py' + + +def load_runner(): + spec = importlib.util.spec_from_file_location('atlas_browser_runner', PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + + +# Finite trusted-unittest assertion helpers. File names and formatted tracebacks +# are never authority for an oracle failure; actual code identities are used. +_ASSERTION_CODES = tuple( + value.__code__ for cls in (unittest.TestCase, unittest.case._AssertRaisesContext, unittest.case._AssertRaisesBaseContext, unittest.case._BaseTestCaseContext) + for name, value in vars(cls).items() + if callable(value) and hasattr(value, '__code__') + and (name.startswith('assert') or name in + ('fail', '_baseAssertEqual', '_formatMessage', '__exit__', '_raiseFailure')) +) + + +def run_oracle_controls(case_type, methods): + """Run frozen synchronous bodies and capture origin before unittest formats it.""" + expected = [case_type(method).id() for method in methods] + if not expected or len(set(expected)) != len(expected): + raise ValueError('nonempty unique oracle inventory required') + states = {} + class OriginResult(unittest.TextTestResult): + def __init__(self, *args): + super().__init__(*args) + self.started = [] + self.stopped = [] + self.origins = [] + self.successful = [] + def startTest(self, test): + self.started.append(test.id()) + super().startTest(test) + def stopTest(self, test): + self.stopped.append(test.id()) + super().stopTest(test) + def addSuccess(self, test): + self.successful.append(test.id()) + super().addSuccess(test) + def capture(self, test, err): + state = states.get(test.id(), {}) + codes = [] + tb = err[2] + while tb is not None: + codes.append(tb.tb_frame.f_code) + tb = tb.tb_next + body = state.get('code') + positions = [i for i, code in enumerate(codes) if code is body] + valid = (type(err[1]) is AssertionError and state.get('entered') == 1 + and state.get('phase') in ('body', 'body-exited') + and len(positions) == 1 + and all(any(code is allowed for allowed in _ASSERTION_CODES) for code in codes[positions[0]+1:])) + self.origins.append((test.id(), valid)) + def addFailure(self, test, err): + self.capture(test, err) + super().addFailure(test, err) + def addSubTest(self, test, subtest, err): + if err is not None: + self.capture(test, err) + super().addSubTest(test, subtest, err) + cases = [] + for method in methods: + case = case_type(method) + selected = getattr(case, method) + code = getattr(selected, '__code__', None) + if code is None or inspect.iscoroutinefunction(selected) or inspect.isgeneratorfunction(selected) or inspect.isasyncgenfunction(selected): + raise ValueError('only synchronous oracle bodies are admitted') + state = dict(code=code, entered=0, phase='initial') + states[case.id()] = state + for attr, phase in (('_callSetUp', 'setup'), ('_callTearDown', 'teardown'), ('_callCleanup', 'cleanup')): + original = getattr(case, attr) + def call(*args, _original=original, _phase=phase, _state=state, **kwargs): + _state['phase'] = _phase + return _original(*args, **kwargs) + setattr(case, attr, call) + original = case._callTestMethod + def body_call(method, _original=original, _state=state): + if getattr(method, '__code__', None) is not _state['code']: + raise RuntimeError('oracle body changed') + _state['entered'] += 1 + _state['phase'] = 'body' + try: + returned = method() + if returned is not None: + if inspect.iscoroutine(returned) or inspect.isgenerator(returned): + returned.close() + raise RuntimeError('oracle body returned a deferred or unsupported value') + return None + finally: + _state['phase'] = 'body-exited' + case._callTestMethod = body_call + cases.append(case) + result = unittest.TextTestRunner(stream=io.StringIO(), resultclass=OriginResult).run(unittest.TestSuite(cases)) + complete = (result.testsRun == len(expected) and result.started == expected + and result.stopped == expected and all(s['entered'] == 1 for s in states.values()) + and not result.errors and not result.skipped and not result.expectedFailures + and not result.unexpectedSuccesses) + accounted = set(result.successful) | {identity for identity, unused in result.origins} + complete = (complete and accounted == set(expected) + and len(result.successful) == len(set(result.successful)) + and not (set(result.successful) & {identity for identity, unused in result.origins})) + result.oracle_baseline = complete and result.wasSuccessful() and not result.origins + result.oracle_killed = complete and bool(result.failures) and bool(result.origins) and all(v for _, v in result.origins) + return result + + +class AtlasBrowserRunnerControls(unittest.TestCase): + def setUp(self): + self.runner = load_runner() + + def suite(self, behavior='pass', ids=None): + ids = self.runner.EXPECTED_IDS if ids is None else ids + cases = [] + for index, identity in enumerate(ids): + def test(case, selected=index == 0): + if not selected: + return + if behavior == 'skip': + case.skipTest('synthetic missing prerequisite') + elif behavior in ('failure', 'expected_failure'): + case.fail('synthetic assertion failure') + elif behavior == 'error': + raise RuntimeError('synthetic runtime error') + elif behavior == 'subtest_failure': + with case.subTest(value=1): + case.fail('synthetic subtest failure') + if behavior in ('expected_failure', 'unexpected_success') and index == 0: + test = unittest.expectedFailure(test) + cls = type('SyntheticCase', (unittest.TestCase,), { + 'runTest': test, 'id': lambda case, value=identity: value}) + cases.append(cls()) + return unittest.TestSuite(cases) + + def test_exact_two_successes_pass(self): + result = self.runner.run_suite(self.suite(), io.StringIO()) + self.assertTrue(result['passed']) + self.assertEqual(result['tests_run'], 2) + self.assertEqual(result['successful_ids'], list(self.runner.EXPECTED_IDS)) + + def test_empty_inventory_refused(self): + with self.assertRaisesRegex(ValueError, 'test inventory'): + self.runner.run_suite(self.suite(ids=[]), io.StringIO()) + + def test_duplicate_inventory_refused(self): + with self.assertRaisesRegex(ValueError, 'test inventory'): + self.runner.run_suite(self.suite(ids=[self.runner.EXPECTED_IDS[0]] * 2), io.StringIO()) + + def test_wrong_and_extra_ids_refused(self): + for ids in [('wrong', self.runner.EXPECTED_IDS[1]), (*self.runner.EXPECTED_IDS, 'extra')]: + with self.subTest(ids=ids), self.assertRaisesRegex(ValueError, 'test inventory'): + self.runner.run_suite(self.suite(ids=ids), io.StringIO()) + + def test_skip_is_failure(self): + result = self.runner.run_suite(self.suite('skip'), io.StringIO()) + self.assertFalse(result['passed']) + self.assertEqual(len(result['skips']), 1) + + def test_failures_errors_and_subtests_are_not_success(self): + for kind in ('failure', 'error', 'subtest_failure'): + with self.subTest(kind=kind): + self.assertFalse(self.runner.run_suite(self.suite(kind), io.StringIO())['passed']) + + def test_expected_failure_is_not_success(self): + result = self.runner.run_suite(self.suite('expected_failure'), io.StringIO()) + self.assertFalse(result['passed']) + self.assertEqual(result['expected_failures'], 1) + + def test_unexpected_success_is_not_success(self): + result = self.runner.run_suite(self.suite('unexpected_success'), io.StringIO()) + self.assertFalse(result['passed']) + self.assertEqual(result['unexpected_successes'], 1) + + def test_missing_execution_is_not_success(self): + class NoExecution(unittest.TestSuite): + def run(self, result, debug=False): + return result + suite = NoExecution(list(self.suite())) + self.assertFalse(self.runner.run_suite(suite, io.StringIO())['passed']) + + def invoke_main(self, directory, behavior='pass', identities=None): + identities = identities or [{'commit': 'a', 'tree': 'b'}] * 2 + with mock.patch.object(self.runner, 'git_identity', side_effect=identities), \ + mock.patch.object(self.runner, 'runtime_identity', return_value={'synthetic': True}), \ + mock.patch.object(self.runner.unittest.defaultTestLoader, 'loadTestsFromNames', return_value=self.suite(behavior)), \ + mock.patch.dict(self.runner.os.environ), \ + mock.patch.object(self.runner.sys, 'path', list(self.runner.sys.path)), \ + mock.patch.object(self.runner.sys, 'stderr', io.StringIO()): + return self.runner.main(['--output', str(directory)]) + + def test_main_success_and_failure_exit_match_new_report(self): + for behavior, expected in [('pass', 0), ('skip', 1), ('error', 1), ('expected_failure', 1)]: + with self.subTest(behavior=behavior), tempfile.TemporaryDirectory() as temp: + output = Path(temp) / 'attempt' + self.assertEqual(self.invoke_main(output, behavior), expected) + report = json.loads((output / 'result.json').read_text()) + self.assertEqual(report['passed'], expected == 0) + self.assertEqual(report['scientific_effect'], 'NONE') + + def test_reused_output_is_refused_without_overwrite(self): + with tempfile.TemporaryDirectory() as temp: + output = Path(temp) + previous = b'original retained bytes\n' + (output / 'result.json').write_bytes(previous) + with self.assertRaises(FileExistsError): + self.invoke_main(output) + self.assertEqual((output / 'result.json').read_bytes(), previous) + + def test_source_drift_cannot_leave_successful_report(self): + with tempfile.TemporaryDirectory() as temp: + output = Path(temp) / 'attempt' + identities = [{'commit': 'a', 'tree': 'b'}, {'commit': 'c', 'tree': 'd'}] + self.assertEqual(self.invoke_main(output, identities=identities), 1) + report = json.loads((output / 'result.json').read_text()) + self.assertFalse(report['passed']) + self.assertIn('source identity changed', report['error']) + + def test_six_runner_source_mutants_are_killed_by_assertions(self): + source = PATH.read_text() + mutations = ( + ('inventory', 'if tuple(test_ids(suite)) != EXPECTED_IDS:', 'if False:', 'test_empty_inventory_refused'), + ('green', "'passed': passed", "'passed': True", 'test_skip_is_failure'), + ('drift', "if report['source_before'] != report['source_after']:", 'if False:', 'test_source_drift_cannot_leave_successful_report'), + ('overwrite', 'output.mkdir(parents=True, exist_ok=False)', 'output.mkdir(parents=True, exist_ok=True)', 'test_reused_output_is_refused_without_overwrite'), + ('exit', "return 0 if report['passed'] else 1", 'return 0', 'test_main_success_and_failure_exit_match_new_report'), + ('error', "report['passed'] = False", 'pass', 'test_source_drift_cannot_leave_successful_report'), + ) + for name, before, after, method in mutations: + with self.subTest(mutation=name), tempfile.TemporaryDirectory() as temp: + self.assertIn(before, source) + path = Path(temp) / 'mutant.py' + path.write_text(source.replace(before, after, 1)) + spec = importlib.util.spec_from_file_location('mutant_' + name, path) + mutant = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mutant) + baseline = run_oracle_controls(type(self), (method,)) + self.assertTrue(baseline.oracle_baseline, name) + with mock.patch(__name__ + '.load_runner', return_value=mutant): + result = run_oracle_controls(type(self), (method,)) + self.assertTrue(result.oracle_killed, name) + self.assertGreater(len(result.failures), 0, name) + self.assertEqual(result.errors, [], name) + self.assertEqual(result.skipped, [], name) + + def test_focus_and_reader_explicitly_enable_sandbox(self): + for name in ('test_atlas_focus_geometry.py', 'test_atlas_reader_scroll.py'): + tree = ast.parse((ROOT / 'tests' / name).read_text()) + launches = [node for node in ast.walk(tree) if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) and node.func.attr == 'launch'] + self.assertEqual(len(launches), 1) + values = {item.arg: item.value for item in launches[0].keywords} + self.assertIn('chromium_sandbox', values) + self.assertIs(values['chromium_sandbox'].value, True) + + def browser_selection(self, filename, attribute, override, available): + tree = ast.parse((ROOT / 'tests' / filename).read_text()) + assignments = [node for node in tree.body if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == attribute + for target in node.targets)] + self.assertEqual(len(assignments), 1) + launches = [node for node in ast.walk(tree) if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) and node.func.attr == 'launch'] + executable = next(item.value for item in launches[0].keywords + if item.arg == 'executable_path') + self.assertEqual(ast.dump(executable), ast.dump(ast.Name(id=attribute, ctx=ast.Load()))) + environment = {} if override is None else {'ATLAS_BROWSER_EXECUTABLE': override} + return eval(compile(ast.Expression(assignments[0].value), filename, 'eval'), { + 'os': types.SimpleNamespace(environ=environment), + 'shutil': types.SimpleNamespace(which=available.get)}) + + def test_both_tests_select_override_then_chrome_then_chromium(self): + cases = ( + ('/explicit/browser', {'google-chrome': '/chrome', 'chromium': '/chromium'}, '/explicit/browser'), + (None, {'google-chrome': '/chrome', 'chromium': '/chromium'}, '/chrome'), + ('', {'google-chrome': '/chrome', 'chromium': '/chromium'}, '/chrome'), + (None, {'chromium': '/chromium'}, '/chromium'), + (None, {}, None), + ) + for filename, attribute in (('test_atlas_focus_geometry.py', 'CHROMIUM'), + ('test_atlas_reader_scroll.py', 'BROWSER')): + for override, available, expected in cases: + with self.subTest(filename=filename, override=override, available=available): + self.assertEqual(self.browser_selection(filename, attribute, override, available), expected) + + def test_runtime_identity_records_each_test_selected_launcher(self): + # Version probing is stubbed: no real executable or browser is launched. + with tempfile.TemporaryDirectory() as temp: + modules = {} + expected = [] + for filename, attribute in (('test_atlas_focus_geometry.py', 'CHROMIUM'), + ('test_atlas_reader_scroll.py', 'BROWSER')): + launcher = Path(temp) / filename + launcher.write_bytes(filename.encode()) + selected = self.browser_selection(filename, attribute, str(launcher), {}) + self.assertEqual(selected, str(launcher)) + name = 'tests.' + filename.removesuffix('.py') + modules[name] = types.SimpleNamespace(**{attribute: selected}) + expected.append({'module': name, 'path': str(launcher.resolve()), + 'version': 'controlled version', + 'sha256': hashlib.sha256(launcher.read_bytes()).hexdigest()}) + with mock.patch.dict(self.runner.sys.modules, modules), \ + mock.patch.object(self.runner.subprocess, 'check_output', return_value='controlled version\n') as probe, \ + mock.patch.object(self.runner.importlib.metadata, 'version', return_value='controlled playwright'): + result = self.runner.runtime_identity() + self.assertEqual(result['browsers'], expected) + self.assertEqual(probe.call_args_list, [mock.call([item['path'], '--version'], text=True, + stderr=self.runner.subprocess.STDOUT, timeout=15) for item in expected]) + + def test_workflow_runs_both_modes_and_retains_evidence(self): + source = (ROOT / '.github/workflows/public-shop.yml').read_text() + self.assertIn('python -B tools/run_atlas_browser_regressions.py --output browser-evidence/atlas-regressions/normal', source) + self.assertIn('python -B -O tools/run_atlas_browser_regressions.py --output browser-evidence/atlas-regressions/optimized', source) + self.assertIn('tests.test_atlas_browser_regressions_runner', source) + self.assertIn('path: browser-evidence', source) + self.assertIn('if: always()', source) + + +class OracleOriginControls(unittest.TestCase): + def observe(self, body, **attributes): + cls = type('FrozenOracle', (unittest.TestCase,), dict(test_body=body, **attributes)) + return run_oracle_controls(cls, ('test_body',)) + + def test_genuine_baseline_and_assertion_origins(self): + def positive(case): case.assertEqual(1, 1) + def failure(case): case.assertEqual(1, 2) + def subtest(case): + with case.subTest(value=1): case.assertFalse(True) + def context(case): + with case.assertRaises(ValueError): pass + self.assertTrue(self.observe(positive).oracle_baseline) + for body in (failure, subtest, context): + with self.subTest(body=body.__name__): + result = self.observe(body) + self.assertTrue(result.oracle_killed) + self.assertFalse(result.oracle_baseline) + + def test_fixture_assertions_cannot_kill(self): + def positive(case): pass + def failure(case): case.fail('actual body') + def fixture(case): case.fail('fixture') + for attr in ('setUp', 'tearDown'): + for body in (positive, failure): + with self.subTest(attr=attr, body=body.__name__): + result = self.observe(body, **{attr: fixture}) + self.assertFalse(result.oracle_killed) + self.assertFalse(result.oracle_baseline) + + def test_later_phase_calling_selected_body_cannot_kill(self): + def body(case): case.fail('selected body') + def later(case): case.test_body() + self.assertFalse(self.observe(body, tearDown=later).oracle_killed) + def setup(case): case.addCleanup(case.test_body) + self.assertFalse(self.observe(body, setUp=setup).oracle_killed) + + def test_cleanup_assertions_cannot_kill(self): + def cleanup(): raise AssertionError('cleanup') + def body(case): + case.addCleanup(cleanup) + case.fail('body') + self.assertFalse(self.observe(body).oracle_killed) + + def test_subject_helper_callback_assertions_cannot_kill(self): + def subject(): raise AssertionError('subject') + def subject_body(case): subject() + def helper(case): case.fail('helper') + def helper_body(case): case.helper() + def callback_body(case): + callback = lambda: (_ for _ in ()).throw(AssertionError('callback')) + callback() + for body in (subject_body, helper_body, callback_body): + with self.subTest(body=body.__name__): + self.assertFalse(self.observe(body, helper=helper).oracle_killed) + + def test_cloned_approved_code_is_not_authorized_identity(self): + original = unittest.TestCase.fail + cloned_code = original.__code__.replace(co_filename='subject_fake.py') + self.assertIsNot(cloned_code, original.__code__) + self.assertEqual(cloned_code, original.__code__) + cloned = types.FunctionType(cloned_code, original.__globals__, + original.__name__, original.__defaults__) + def body(case): cloned(case, 'cloned subject code') + self.assertFalse(self.observe(body).oracle_killed) + + def test_forged_unittest_filename_is_not_authority(self): + namespace = {} + exec(compile("def forged(): raise AssertionError('forged')", unittest.case.__file__, 'exec'), namespace) + def body(case): namespace['forged']() + self.assertFalse(self.observe(body).oracle_killed) + + def test_setup_calling_body_does_not_count_as_execution(self): + def body(case): case.fail('called from setup') + def setup(case): case.test_body() + self.assertFalse(self.observe(body, setUp=setup).oracle_killed) + + def test_error_skip_expected_failure_refused(self): + def error(case): raise RuntimeError('error') + def skip(case): case.skipTest('skip') + @unittest.expectedFailure + def expected(case): case.fail('expected') + @unittest.expectedFailure + def unexpected(case): pass + for body in (error, skip, expected, unexpected): + with self.subTest(body=body.__name__): + result = self.observe(body) + self.assertFalse(result.oracle_killed) + self.assertFalse(result.oracle_baseline) + + def test_deferred_bodies_refused(self): + async def coroutine(case): pass + def generator(case): yield 1 + async def async_generator(case): yield 1 + for body in (coroutine, generator, async_generator): + with self.subTest(body=body.__name__), self.assertRaisesRegex(ValueError, 'synchronous'): + self.observe(body) + + def test_synchronous_returning_deferred_or_value_refused(self): + async def deferred(): pass + def coroutine(case): return deferred() + def generator(case): return iter((1,)) + def value(case): return 7 + for body in (coroutine, generator, value): + with self.subTest(body=body.__name__): + result = self.observe(body) + self.assertFalse(result.oracle_baseline) + self.assertFalse(result.oracle_killed) + + def test_empty_duplicate_population_refused(self): + cls = type('FrozenOracle', (unittest.TestCase,), {'test_body': lambda case: None}) + for methods in ((), ('test_body', 'test_body')): + with self.subTest(methods=methods), self.assertRaisesRegex(ValueError, 'inventory'): + run_oracle_controls(cls, methods) + + def test_repeated_body_execution_refused(self): + def body(case): case.fail('body') + def run(case, result): + result.startTest(case) + for unused in range(2): + try: case._callTestMethod(case.test_body) + except AssertionError: + import sys + result.addFailure(case, sys.exc_info()) + result.stopTest(case) + self.assertFalse(self.observe(body, run=run).oracle_killed) + + def test_missing_or_duplicate_terminal_success_refused(self): + def body(case): pass + for count in (0, 2): + def run(case, result): + result.startTest(case) + case._callTestMethod(case.test_body) + for unused in range(count): result.addSuccess(case) + result.stopTest(case) + with self.subTest(count=count): + self.assertFalse(self.observe(body, run=run).oracle_baseline) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_atlas_focus_geometry.py b/tests/test_atlas_focus_geometry.py index a7409191e..f9eeaaa08 100644 --- a/tests/test_atlas_focus_geometry.py +++ b/tests/test_atlas_focus_geometry.py @@ -14,7 +14,7 @@ sync_playwright = None ROOT = Path(__file__).resolve().parents[1] -CHROMIUM = shutil.which("chromium") or shutil.which("google-chrome") +CHROMIUM = os.environ.get("ATLAS_BROWSER_EXECUTABLE") or shutil.which("google-chrome") or shutil.which("chromium") @unittest.skipUnless(sync_playwright and CHROMIUM, @@ -31,7 +31,8 @@ def log_message(self, *_args): thread.start() try: with sync_playwright() as playwright: - browser = playwright.chromium.launch(executable_path=CHROMIUM, headless=True) + browser = playwright.chromium.launch(executable_path=CHROMIUM, headless=True, + chromium_sandbox=True) try: page = browser.new_page(viewport={"width": 390, "height": 844}, reduced_motion="reduce") diff --git a/tools/run_atlas_browser_regressions.py b/tools/run_atlas_browser_regressions.py new file mode 100644 index 000000000..3bf3a61ae --- /dev/null +++ b/tools/run_atlas_browser_regressions.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Run exactly two Atlas browser tests; skips are failure, never optional green. + +Use without -S after installing tests/browser-requirements.txt. The output +folder must be new. This is trusted unittest execution, not hostile-code +containment, mathematical certification or a signature of remote execution. +""" +import argparse +import hashlib +import importlib.metadata +import json +import os +from pathlib import Path +import subprocess +import sys +import traceback +import unittest + +ROOT = Path(__file__).resolve().parents[1] +EXPECTED_IDS = ( + 'tests.test_atlas_focus_geometry.AtlasListFocusGeometry.test_keyboard_focus_ring_fits_inside_the_list_scroll_boundary', + 'tests.test_atlas_reader_scroll.AtlasReaderScroll.test_new_selection_after_a_long_read_keeps_its_heading_visible', +) + + +def test_ids(suite): + for item in suite: + if isinstance(item, unittest.TestSuite): + yield from test_ids(item) + else: + yield item.id() + + +class ExactResult(unittest.TextTestResult): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.successful_ids = [] + + def addSuccess(self, test): + self.successful_ids.append(test.id()) + super().addSuccess(test) + + +def run_suite(suite, stream): + if tuple(test_ids(suite)) != EXPECTED_IDS: + raise ValueError('test inventory must equal the two fixed Atlas method IDs') + result = unittest.TextTestRunner(stream=stream, verbosity=2, + resultclass=ExactResult).run(suite) + passed = (result.wasSuccessful() and result.testsRun == 2 + and tuple(result.successful_ids) == EXPECTED_IDS + and not result.skipped and not result.expectedFailures + and not result.unexpectedSuccesses) + return {'passed': passed, 'tests_run': result.testsRun, + 'successful_ids': result.successful_ids, + 'skips': [[test.id(), reason] for test, reason in result.skipped], + 'failures': len(result.failures), 'errors': len(result.errors), + 'expected_failures': len(result.expectedFailures), + 'unexpected_successes': len(result.unexpectedSuccesses)} + + +def git_identity(): + def git(*args): + return subprocess.check_output(['git', *args], cwd=ROOT, text=True).strip() + if git('status', '--porcelain', '--untracked-files=no'): + raise ValueError('tracked source is dirty') + return {'commit': git('rev-parse', 'HEAD'), 'tree': git('rev-parse', 'HEAD^{tree}')} + + +def runtime_identity(): + browsers = [] + for module_name, attribute in ( + ('tests.test_atlas_focus_geometry', 'CHROMIUM'), + ('tests.test_atlas_reader_scroll', 'BROWSER')): + executable = getattr(sys.modules[module_name], attribute) + if not executable: + raise ValueError('required packaged browser executable is unavailable') + path = Path(executable).resolve(strict=True) + version = subprocess.check_output([str(path), '--version'], text=True, + stderr=subprocess.STDOUT, timeout=15).strip() + browsers.append({'module': module_name, 'path': str(path), 'version': version, + 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}) + return {'python': sys.version, 'optimize': sys.flags.optimize, + 'playwright': importlib.metadata.version('playwright'), 'browsers': browsers} + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, required=True) + output = parser.parse_args(argv).output.resolve() + # Refuse reuse before any browser/test work; retain earlier attempts intact. + output.mkdir(parents=True, exist_ok=False) + report = {'schema': 'atlas-browser-regressions/1', 'passed': False, + 'expected_ids': list(EXPECTED_IDS), 'scientific_effect': 'NONE'} + try: + report['source_before'] = git_identity() + sys.path.insert(0, str(ROOT)) + os.environ['ATLAS_FOCUS_EVIDENCE'] = str(output / 'focus') + os.environ['ATLAS_READER_EVIDENCE'] = str(output / 'reader') + suite = unittest.defaultTestLoader.loadTestsFromNames(EXPECTED_IDS) + # Loader errors and absent prerequisites must not become a skip-green. + if tuple(test_ids(suite)) != EXPECTED_IDS: + raise ValueError('test inventory must equal the two fixed Atlas method IDs') + report['runtime'] = runtime_identity() + report.update(run_suite(suite, sys.stderr)) + report['source_after'] = git_identity() + if report['source_before'] != report['source_after']: + raise ValueError('source identity changed during browser tests') + except Exception as error: + report['passed'] = False + report['error'] = f'{type(error).__name__}: {error}' + traceback.print_exc() + finally: + (output / 'result.json').write_text(json.dumps(report, indent=2, sort_keys=True) + '\n') + return 0 if report['passed'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main())