diff --git a/.github/workflows/cbz-auditor-origin.yml b/.github/workflows/cbz-auditor-origin.yml new file mode 100644 index 000000000..3dd949eaf --- /dev/null +++ b/.github/workflows/cbz-auditor-origin.yml @@ -0,0 +1,80 @@ +name: CBZ auditor assertion-origin controls +'on': + push: + branches: [main] + paths: + - '.github/workflows/cbz-auditor-origin.yml' + - 'tests/cbz_auditor_origin*' + - 'tests/cbz_inventory_origin.py' + - 'tests/test_cbz_auditor_origin*' + - 'tests/test_cbz_inventory_origin.py' + - 'tests/test_cbz_layer_a_scan*' + - 'tests/test_cbz_a3_float_census.py' + - 'tests/test_cbz_auditor_hardening.py' + - 'tools/cbz/layer_a_scan.py' + - 'tools/cbz/a3_float_census.py' + pull_request: + paths: + - '.github/workflows/cbz-auditor-origin.yml' + - 'tests/cbz_auditor_origin*' + - 'tests/cbz_inventory_origin.py' + - 'tests/test_cbz_auditor_origin*' + - 'tests/test_cbz_inventory_origin.py' + - 'tests/test_cbz_layer_a_scan*' + - 'tests/test_cbz_a3_float_census.py' + - 'tests/test_cbz_auditor_hardening.py' + - 'tools/cbz/layer_a_scan.py' + - 'tools/cbz/a3_float_census.py' + workflow_dispatch: +permissions: + contents: read +jobs: + origin-controls: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 + with: + python-version: '3.11.16' + - name: Run both actual modes and retain each mutation observation + shell: bash + run: | + set -uo pipefail + evidence="$RUNNER_TEMP/cbz-auditor-origin" + mkdir "$evidence" || exit 1 + git rev-parse HEAD > "$evidence/commit.txt" || exit 1 + git status --porcelain=v1 --untracked-files=all > "$evidence/worktree-before.txt" || exit 1 + if [ -s "$evidence/worktree-before.txt" ]; then echo 'source checkout is not clean' >&2; exit 1; fi + python --version > "$evidence/python.txt" 2>&1 || exit 1 + sha256sum .github/workflows/cbz-auditor-origin.yml tests/cbz_auditor_origin.py tests/cbz_auditor_origin_mutation_child.py tests/cbz_inventory_origin.py tests/test_cbz_auditor_origin.py tests/test_cbz_auditor_origin_mutations.py tests/test_cbz_auditor_origin_workflow.py tests/test_cbz_inventory_origin.py tests/test_cbz_layer_a_scan.py tests/test_cbz_layer_a_scan_mutants.py tests/test_cbz_auditor_hardening.py tests/test_cbz_a3_float_census.py tools/cbz/layer_a_scan.py tools/cbz/a3_float_census.py > "$evidence/source-sha256.txt" || exit 1 + failed=0 + for mode in normal optimized; do + mkdir "$evidence/$mode" "$evidence/$mode/census" "$evidence/$mode/adapter" || exit 1 + flags=(-B -S) + if [ "$mode" = optimized ]; then flags=(-B -O -S); fi + if CBZ_AUDITOR_ORIGIN_MUTATION_EVIDENCE_DIR="$evidence/$mode/adapter" python "${flags[@]}" -m unittest tests.test_cbz_auditor_origin tests.test_cbz_inventory_origin tests.test_cbz_auditor_origin_mutations tests.test_cbz_auditor_origin_workflow -v > "$evidence/$mode/origin.log" 2>&1; then status=0; else status=$?; failed=1; fi + printf '%s\n' "$status" > "$evidence/$mode/origin.exit" || failed=1 + if CBZ_AUDITOR_EVIDENCE_DIR="$evidence/$mode/census" python "${flags[@]}" -m unittest tests.test_cbz_layer_a_scan tests.test_cbz_auditor_hardening tests.test_cbz_layer_a_scan_mutants tests.test_cbz_a3_float_census -v > "$evidence/$mode/census.log" 2>&1; then status=0; else status=$?; failed=1; fi + printf '%s\n' "$status" > "$evidence/$mode/census.exit" || failed=1 + grep -Eq '^Ran 22 tests in ' "$evidence/$mode/origin.log" || failed=1 + grep -Eq '^Ran 83 tests in ' "$evidence/$mode/census.log" || failed=1 + grep -Fxq 'OK' "$evidence/$mode/origin.log" || failed=1 + grep -Fxq 'OK' "$evidence/$mode/census.log" || failed=1 + if grep -Fq ' ... skipped ' "$evidence/$mode/origin.log" "$evidence/$mode/census.log"; then failed=1; fi + [ "$(find "$evidence/$mode/adapter" -name '*.json' -type f | wc -l)" -eq 10 ] || failed=1 + [ "$(find "$evidence/$mode/census" -maxdepth 1 -name '*.json' -type f | wc -l)" -eq 202 ] || failed=1 + done + sha256sum -c "$evidence/source-sha256.txt" > "$evidence/source-readback.log" 2>&1 || failed=1 + exit "$failed" + - name: Preserve complete observations including failures + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: cbz-auditor-origin-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/cbz-auditor-origin + include-hidden-files: true + if-no-files-found: error + retention-days: 90 diff --git a/BASELINE.md b/BASELINE.md index 235f0b747..0e114a713 100644 --- a/BASELINE.md +++ b/BASELINE.md @@ -1,5 +1,20 @@ # CBZ baseline ledger +## Auditor-origin candidate — 9 November 2026 + +The observed published cut has Main `05f2289478a0ee9492adc1eb501466e0f66866df`, +Math `9bdfc54b76173b57912503bfcf901a5f9d43b09c` and Workspace +`bda15fcd29d6d484f9b2e1233663dbd8fdd8b796`. The Workspace pin and contract +runs passed their bounded controls, and the Main normal discovery retained +24 skips. A later Main merge needs a fresh guarded Workspace pin update. + +Two Main auditor mutation graders were observed crediting subject assertions +as kills. The source repair and its actual current-head tests are a candidate, +not a final G0/G1 or Layer A clearance. The intentional negative axiom fixture, +historical unparseable Python writer and Workspace gitlink refusal remain +visible in the scoped scan evidence. No CBZ-0 or terminal Partial is declared. + + **State: active intermediate baseline; CBZ-0 is not established. No terminal Partial Baseline is declared.** This ledger binds the currently observed repository snapshot to the twelve diff --git a/DECISIONS.md b/DECISIONS.md index 3015a4854..5853e9f8e 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -172,3 +172,27 @@ Append-only. Historical source outcomes remain in their original records. engineering test, not continuum closure or independent scientific review. Its disposition is reversible through Git history; prior outcomes remain preserved. + + +## 2026-11-09 — auditor assertion-origin repair + +- Preserve the two real Main grader false kills in each Python mode at + `05f2289478a0ee9492adc1eb501466e0f66866df`, their raw child streams and + the failed local attempts in [the evidence record](cbz/evidence/auditor-origin-20261109/README.md). + A subject assertion is not an authorized oracle outcome. +- Reuse the existing pre-cleanup live-code recorder for each designated test, + require an unchanged-source clean baseline and reject missing provenance, + errors, undeclared skips or source drift. Require actual source mutants and + adverse controls in both modes; keep declared inner recursion skips visible. + Tighten the synthetic formatted-traceback-only expectation to refusal. This + changes no scanner subject, theorem, bound or historical result. +- Record the scoped Main negative-axiom fixture finding, Math's historical + unparseable documentary snapshot and Workspace's gitlink refusal. Do not + suppress or rewrite any of them to obtain a green audit. Exact-head hosted + qualification remains necessary; the action is reversible in Git. + +- A later synthetic-producer probe of the actual workflow shell found that an + exit-status evidence write could fail while the shell still returned 0. + Preserve the red output in the evidence record and fail the job on either + metadata-write error; test both guard removals in each Python mode. This + controls evidence completeness, not mathematical truth or scientific status. diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 2a11aacaa..683645013 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -1,5 +1,18 @@ # CBZ-0 governance +## Auditor assertion-origin boundary — 9 November 2026 + +A mutation kill requires a clean designated baseline and an actual failure +from the selected test body or an explicitly authorized standard unittest +assertion helper. Capture live traceback code provenance before unittest +cleanup and reject missing provenance, subject/helper failures, unexpected +skips, errors and source drift. The recorder's supported domain is trusted +synchronous unittest, not hostile code. Historical filename-based false kills +remain preserved and cannot qualify a future gate. Candidate commands and +limits are registered in [TOOLING.md](TOOLING.md#auditor-origin-controls); +current-head tests and the final federation G0–G11 pass remain required. + + The later Math mutation-grader identity defect and its bounded draft correction are recorded in [STATUS.md](STATUS.md#later-node-inventory-and-pin-observations--9-november-2026). Their historical kernel evidence and scientific limits are unchanged. diff --git a/PROOF_INDEX.md b/PROOF_INDEX.md index befe2070b..477050ceb 100644 --- a/PROOF_INDEX.md +++ b/PROOF_INDEX.md @@ -1,5 +1,16 @@ # CBZ proof and quarantine index +## Auditor-origin boundary — 9 November 2026 + +The Main and Workspace engineering changes and the auditor-grader repair do +not modify any theorem, proof disposition, authorship or scientific claim. +The unapproved axiom in `cbz/contributions/math514/kernel_replay/tests/fixtures/UnapprovedAxiom.lean` +is a deliberate negative test fixture, not a closed mathematical module; its +scanner finding remains recorded. The bounded lexical census does not prove +its transitive isolation or establish a complete axiom audit. Historical Math +conditional kernel results and continuum limits remain as recorded below. + + **State: incomplete inventory; CBZ-0 is not established.** This is the Main cold-start map for proof status. It records exact boundaries; diff --git a/STATUS.md b/STATUS.md index 4ca409b5c..d57e56e2c 100644 --- a/STATUS.md +++ b/STATUS.md @@ -1,5 +1,34 @@ # CBZ-0 execution status +## Auditor-origin repair and published pin cut — 9 November 2026 + +Main #791 merged as `05f2289478a0ee9492adc1eb501466e0f66866df`; its hosted +navigation run executed 2,609 tests with 24 skips, and the selected historical +#525 binding test passed in each mode. Workspace #62 merged as +`bda15fcd29d6d484f9b2e1233663dbd8fdd8b796`, pinning that Main commit and Math +`9bdfc54b76173b57912503bfcf901a5f9d43b09c`. The Workspace tested and +postmerge contract runs each executed 254 tests per mode without top-level +skips; pin checks covered eight current pins and four canonical inputs. These +sequential observations do not establish final G5 or G7. + +At Main `05f22894`, actual normal and optimized subject-origin probes showed +that both Lean and Python-census mutation graders could count a subject's +`AssertionError` as a test-oracle kill when the subject was under `tests/`. +The original failures and source identities are preserved in +[the evidence record](cbz/evidence/auditor-origin-20261109/README.md). This +candidate uses live pre-cleanup code provenance and requires a clean baseline +for each source mutant; current-head qualification remains required. Earlier +mutation counts are historical observations, not sound final G0 credit. + +The scoped Lean lexical scans found a deliberately unapproved Main test axiom, +an `unsafe` review token and no listed tokens in 1,974 Math Lean blobs; the +Workspace scan refused a gitlink. The Math Python census failed on one +intentionally preserved historical writer parse error. These are scoped +observations, not whole Layer A clearance or proof that float signals certify +bounds. Exact commands and limits are in [TOOLING.md](TOOLING.md#auditor-origin-controls). +No CBZ-0, terminal Partial or scientific acceptance follows. + + **State: active intermediate baseline; CBZ-0 is not established. No terminal Partial Baseline is declared.** This entry adopts the owner's expanded [Master Directive](governance/MASTER_DIRECTIVE_20261029_SWARM.txt) diff --git a/TOOLING.md b/TOOLING.md index 3c5fdba41..967a484da 100644 --- a/TOOLING.md +++ b/TOOLING.md @@ -1,5 +1,40 @@ # CBZ mission-tool registry +## Auditor origin controls + +**Purpose and scope.** `tests/cbz_auditor_origin.py` adapts the existing +`tests/cbz_inventory_origin.py` pre-cleanup recorder for the Lean lexical and +Python A3 census mutation suites. It admits only selected synchronous test +bodies and the recorder's finite standard-unittest oracle set; it is not a +hostile-child sandbox, a mathematical proof or a complete federation auditor. +For each selected mutant it records unchanged subject-byte hashes and requires +a clean selected baseline. The two historical full-suite baselines explicitly +check and retain their two and four inner recursion-guard skips, respectively. + +**Commands.** From a clean Main checkout, with new evidence directories outside +the checkout (omit either variable only when retained child evidence is not +needed), run each group in normal and optimized mode: + +```sh +mkdir /absolute/new-census-evidence /absolute/new-adapter-evidence +CBZ_AUDITOR_ORIGIN_MUTATION_EVIDENCE_DIR=/absolute/new-adapter-evidence python3 -B -S -m unittest tests.test_cbz_auditor_origin tests.test_cbz_inventory_origin tests.test_cbz_auditor_origin_mutations tests.test_cbz_auditor_origin_workflow -v +CBZ_AUDITOR_EVIDENCE_DIR=/absolute/new-census-evidence python3 -B -S -m unittest tests.test_cbz_layer_a_scan tests.test_cbz_auditor_hardening tests.test_cbz_layer_a_scan_mutants tests.test_cbz_a3_float_census -v +# Repeat both commands with -O and separate new evidence directories. +``` + +**Verification.** The preserved pre-repair real subject-origin failures are +[here](cbz/evidence/auditor-origin-20261109/README.md). The added controls +exercise clean subjects, actual selected-body failures, direct and nested +subject failures, changed source, missing provenance, forged text and +failed evidence writes. Five adapter source mutations have designated +clean-baseline and actual-failure controls; the two existing source campaigns +contain 74 Lean and 27 A3 mutations. The dedicated workflow preserves original +child reports and source snapshots in both modes. Its actual shell guards are +also exercised with synthetic child outcomes and seven guard-removal controls. Actual exact-head hosted +execution and independent readback are required before qualification; historical +pilot observations do not establish final G0, G1 or scientific acceptance. + + **State: component registry; the final federation toolchain is incomplete.** This file lists runnable mission-relevant tools present in Main. The latest diff --git a/cbz/evidence/auditor-origin-20261109/README.md b/cbz/evidence/auditor-origin-20261109/README.md new file mode 100644 index 000000000..bef43a2e4 --- /dev/null +++ b/cbz/evidence/auditor-origin-20261109/README.md @@ -0,0 +1,67 @@ +# Auditor assertion-origin repair: preserved observations + +The base Main commit is `05f2289478a0ee9492adc1eb501466e0f66866df`. At that +commit the Lean grader was blob `faab4ff7e4b515320a95238168ed2d641c226677` +and the A3 grader was blob `b7b56615fa6b11b88d74800478e156be883bedc5`. +Both treated the innermost *formatted traceback filename* as origin authority. +In actual subprocess probes on CPython 3.12.14, an unchanged subject passed, +while an `AssertionError` raised by the subject itself under a directory named +`tests` was incorrectly accepted as a kill by each grader in each Python mode. +`red-results-*.json`, `*-subject-*.stderr` and `initial-probe.py.txt` preserve +those observations and source hashes. `raw-failure-logs.zip` preserves the exact `red-controls-*.log` red +adverse tests and hashes. These failures affect the graders' mutation-credit soundness; +they do not refute a mathematical statement or erase earlier executions. + +The repair uses the existing `tests/cbz_inventory_origin.py` recorder, which +captures actual traceback code identities before unittest cleans the exception. +A selected synchronous test-body failure is a kill only if all later frames +are explicitly authorized standard unittest assertion helpers. A clean +selected-test baseline is required for each actual source mutant. Subject +bytes are hashed before and after the run. Missing or inconsistent provenance, +subject/helper/nested callback exceptions, unexpected skips, errors and source +drift are non-kills. The recorder is for trusted synchronous unittest tests; it +is not a hostile-code sandbox, a Lean kernel, a proof checker or a federation +certificate. Existing historical mutation outcomes are not rebound. + +The initial adapter self-mutation attempt failed in both modes because the +copied adapter inferred the temporary directory as its test checkout; full +failed logs are retained inside `raw-failure-logs.zip`; the explicit +limitations note is in `attempts.log`. The child then explicitly binds the copied helper to the +checkout owning the designated tests. `full-evidence-red.log` preserves the +subsequent red control for missing full-suite evidence. One partial-state log +was accidentally overwritten; its known exit and that loss are recorded in +`attempts.log`, not reconstructed. + +The separately recorded pre-repair lexical scan returned exit 1 on Main's +intentional, unapproved kernel-replay axiom fixture, and reported the replay +adapter's `unsafe` token for review; `main-lexical.json` is unchanged. The +Math scan at `9bdfc54b76173b57912503bfcf901a5f9d43b09c` returned exit 0 +for 1,974 Lean blobs with no listed tokens; `math-lexical.json` is the raw +report. These scans alone cannot establish axiom closure or scientific +alignment. Workspace candidate `e9b6ef598ae249120be0d7498c6d5adcc95a1cab` +returned exit 2 on its gitlink; it was not silently skipped. The Math Python +census returned exit 1 on the preserved historical documentary writer +`frontiers/source_derivative_gaussian_20261025/actual-source-finite-superlevel-components-evidence/raw/producer/history/freeze_documents.001.py` +(blob `f588617889ed04eb2c76bb1ab428fcfa5217ac4b`), whose original parser +failure and later correction are preserved in that packet. Python AST float +signals are a census, not a verdict that a bound is or is not certified. + +The first synthetic workflow-guard control failed because its stub expected +21 tests while the candidate shell still expected 20; that red output remains +in `raw-failure-logs.zip` as `workflow-guards-red-normal.log`, and the workflow now includes the guard +control in its group (now 22 tests). Its seven guard removals use synthetic child +outcomes and do not claim real scanner or mathematical execution. + +A later actual shell-guard probe showed that an unwritable exit-status file +could still produce exit 0 even though the write failed. The two raw +synthetic-producer failures are retained unchanged in `exit-metadata-red.zip`. +The candidate now fails if either exit-status evidence write fails. + +The candidate workflow runs the explicit controls and mutation suites in +both Python modes, retains selected child evidence and full-suite logs even +on failure, and checks source hashes. Its hosted results must be observed on +the exact head before this repair can receive current source credit. The +legacy full-suite baseline intentionally records two Lean and four A3 inner +recursion-guard skips; the exact expected skip identities are checked and +retained, while all 101 selected mutation baselines per mode must have no skips. +This is not the final G0/G1 or a full Layer A audit. diff --git a/cbz/evidence/auditor-origin-20261109/a3-subject-normal.stderr b/cbz/evidence/auditor-origin-20261109/a3-subject-normal.stderr new file mode 100644 index 000000000..da418fe1c --- /dev/null +++ b/cbz/evidence/auditor-origin-20261109/a3-subject-normal.stderr @@ -0,0 +1,20 @@ +test_float_and_complex_calls (tests.test_cbz_a3_float_census.Signals.test_float_and_complex_calls) ... FAIL + +====================================================================== +FAIL: test_float_and_complex_calls (tests.test_cbz_a3_float_census.Signals.test_float_and_complex_calls) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/workspace/scratch/7abea556a9be/federation-vector-post-integration-20261109/main/tests/test_cbz_a3_float_census.py", line 48, in test_float_and_complex_calls + self.assertEqual(kinds("a = float(x)\nb = complex(1, 2)\n"), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/workspace/scratch/7abea556a9be/federation-vector-post-integration-20261109/main/tests/test_cbz_a3_float_census.py", line 34, in kinds + r = M.census_text(src, "x.py") + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/workspace/scratch/7abea556a9be/auditor-origin-probe-20261109/tests/a3-subject_assertion-normal.py", line 311, in census_text + raise AssertionError("ACTUAL_SUBJECT_ORIGIN_PROBE") +AssertionError: ACTUAL_SUBJECT_ORIGIN_PROBE + +---------------------------------------------------------------------- +Ran 1 test in 0.000s + +FAILED (failures=1) diff --git a/cbz/evidence/auditor-origin-20261109/a3-subject-optimized.stderr b/cbz/evidence/auditor-origin-20261109/a3-subject-optimized.stderr new file mode 100644 index 000000000..ec76eb046 --- /dev/null +++ b/cbz/evidence/auditor-origin-20261109/a3-subject-optimized.stderr @@ -0,0 +1,20 @@ +test_float_and_complex_calls (tests.test_cbz_a3_float_census.Signals.test_float_and_complex_calls) ... FAIL + +====================================================================== +FAIL: test_float_and_complex_calls (tests.test_cbz_a3_float_census.Signals.test_float_and_complex_calls) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/workspace/scratch/7abea556a9be/federation-vector-post-integration-20261109/main/tests/test_cbz_a3_float_census.py", line 48, in test_float_and_complex_calls + self.assertEqual(kinds("a = float(x)\nb = complex(1, 2)\n"), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/workspace/scratch/7abea556a9be/federation-vector-post-integration-20261109/main/tests/test_cbz_a3_float_census.py", line 34, in kinds + r = M.census_text(src, "x.py") + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/workspace/scratch/7abea556a9be/auditor-origin-probe-20261109/tests/a3-subject_assertion-optimized.py", line 311, in census_text + raise AssertionError("ACTUAL_SUBJECT_ORIGIN_PROBE") +AssertionError: ACTUAL_SUBJECT_ORIGIN_PROBE + +---------------------------------------------------------------------- +Ran 1 test in 0.000s + +FAILED (failures=1) diff --git a/cbz/evidence/auditor-origin-20261109/attempts.log b/cbz/evidence/auditor-origin-20261109/attempts.log new file mode 100644 index 000000000..62608072e --- /dev/null +++ b/cbz/evidence/auditor-origin-20261109/attempts.log @@ -0,0 +1,6 @@ +The first local scripted replacement updated the Lean grader and then failed with ValueError: substring not found while locating the decorated A3 class. It did not update the A3 grader or the synthetic fixture expectation. The subsequently invoked control suite exited 1 on this partial state. This record preserves that failed implementation attempt; no mutation result from it is qualified. +The initial partial-state suite output was subsequently overwritten by the rerun; only its exit status and this transcription remain. It is not represented as a retained full log. + +The first separate adapter self-mutation campaign ran five attempted baseline controls per mode and failed all five. The copied helper inferred the temporary directory as ROOT, so selected tests could not import the checkout's tests package; the authorized outer assertions correctly failed the baseline. Original reports, copied sources and logs remain in adapter-mutations-first-{normal,optimized} and adapter-campaign-first-{normal,optimized}. The worker now explicitly binds the copied helper to the checkout owning the actual test bodies before rerunning. +A subsequent attempt to write campaign-readback.txt used a path relative to the Main worktree instead of scratch. Its redirect, cat and sha256sum failed with 'No such file or directory'; the shell's later git diff check returned 0, so that command's overall status was not evidence of readback success. The verification was rerun with absolute paths. +An attempt to remove newly staged copies of five raw logs after archiving them was refused by git because they were staged. Their byte-identical, checksum-verified ZIP and original scratch copies were already preserved; the copies were then unstaged and removed, leaving the ZIP as the recoverable native evidence without rewriting the logs. diff --git a/cbz/evidence/auditor-origin-20261109/exit-metadata-red.zip b/cbz/evidence/auditor-origin-20261109/exit-metadata-red.zip new file mode 100644 index 000000000..7d1bd036e Binary files /dev/null and b/cbz/evidence/auditor-origin-20261109/exit-metadata-red.zip differ diff --git a/cbz/evidence/auditor-origin-20261109/full-evidence-red.log b/cbz/evidence/auditor-origin-20261109/full-evidence-red.log new file mode 100644 index 000000000..9057025a6 --- /dev/null +++ b/cbz/evidence/auditor-origin-20261109/full-evidence-red.log @@ -0,0 +1,14 @@ +test_evidence_is_source_bound_and_write_failures_are_not_hidden (tests.test_cbz_auditor_origin.AuditorOriginControls.test_evidence_is_source_bound_and_write_failures_are_not_hidden) ... FAIL + +====================================================================== +FAIL: test_evidence_is_source_bound_and_write_failures_are_not_hidden (tests.test_cbz_auditor_origin.AuditorOriginControls.test_evidence_is_source_bound_and_write_failures_are_not_hidden) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/workspace/scratch/7abea556a9be/cbz-main-g1-origin/tests/test_cbz_auditor_origin.py", line 115, in test_evidence_is_source_bound_and_write_failures_are_not_hidden + self.assertEqual(len(records), 1) +AssertionError: 0 != 1 + +---------------------------------------------------------------------- +Ran 1 test in 2.373s + +FAILED (failures=1) diff --git a/cbz/evidence/auditor-origin-20261109/initial-probe.py.txt b/cbz/evidence/auditor-origin-20261109/initial-probe.py.txt new file mode 100644 index 000000000..41e87abaf --- /dev/null +++ b/cbz/evidence/auditor-origin-20261109/initial-probe.py.txt @@ -0,0 +1,22 @@ +from pathlib import Path +import hashlib, importlib, json, sys +root=Path('/workspace/scratch/7abea556a9be/federation-vector-post-integration-20261109/main') +base=Path('/workspace/scratch/7abea556a9be/auditor-origin-probe-20261109') +sys.path.insert(0,str(root)) +cases=[('lean','tests.test_cbz_layer_a_scan_mutants','tools/cbz/layer_a_scan.py','test_native_decide_options','scan_text'),('a3','tests.test_cbz_a3_float_census','tools/cbz/a3_float_census.py','test_float_and_complex_calls','census_text')] +mode='optimized' if sys.flags.optimize else 'normal';rows=[] +for label,module,source,test,fn in cases: + grader=importlib.import_module(module);orig=(root/source).read_bytes();test_id=grader.designated_test_ids()[test] + for kind in ['baseline','subject_assertion']: + p=base/'tests'/(label+'-'+kind+'-'+mode+'.py') + body=orig if kind=='baseline' else orig+('\n\ndef '+fn+'(*args, **kwargs):\n raise AssertionError("ACTUAL_SUBJECT_ORIGIN_PROBE")\n').encode() + p.write_bytes(body) + r=grader.run_suite(str(p),test_id);accepted=grader.killed(r,test) + stem=base/(label+'-'+kind+'-'+mode) + stem.with_suffix('.stdout').write_text(r.stdout);stem.with_suffix('.stderr').write_text(r.stderr) + record={'grader':module,'mode':mode,'source':source,'test_id':test_id,'kind':kind,'subject_path':str(p),'original_sha256':hashlib.sha256(orig).hexdigest(),'subject_sha256':hashlib.sha256(body).hexdigest(),'exit_code':r.returncode,'accepted_as_kill':accepted,'stdout_sha256':hashlib.sha256(r.stdout.encode()).hexdigest(),'stderr_sha256':hashlib.sha256(r.stderr.encode()).hexdigest()} + rows.append(record) + if kind=='baseline' and (r.returncode!=0 or accepted):raise RuntimeError('baseline did not pass or was credited as kill') + if kind=='subject_assertion' and (r.returncode!=1 or 'ACTUAL_SUBJECT_ORIGIN_PROBE' not in r.stderr):raise RuntimeError('actual subject-origin failure not observed') +(base/('results-'+mode+'.json')).write_text(json.dumps(rows,indent=2)+'\n') +for r in rows:print(r['grader'],r['kind'],'exit',r['exit_code'],'accepted_as_kill',r['accepted_as_kill']) diff --git a/cbz/evidence/auditor-origin-20261109/lean-subject-normal.stderr b/cbz/evidence/auditor-origin-20261109/lean-subject-normal.stderr new file mode 100644 index 000000000..5f0c75a3c --- /dev/null +++ b/cbz/evidence/auditor-origin-20261109/lean-subject-normal.stderr @@ -0,0 +1,20 @@ +test_native_decide_options (tests.test_cbz_layer_a_scan.ViolationTokens.test_native_decide_options) ... FAIL + +====================================================================== +FAIL: test_native_decide_options (tests.test_cbz_layer_a_scan.ViolationTokens.test_native_decide_options) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/workspace/scratch/7abea556a9be/federation-vector-post-integration-20261109/main/tests/test_cbz_layer_a_scan.py", line 64, in test_native_decide_options + self.assertEqual(hits("example : 2 = 2 := by decide +native"), [(1, 31, "native", "violation")]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/workspace/scratch/7abea556a9be/federation-vector-post-integration-20261109/main/tests/test_cbz_layer_a_scan.py", line 32, in hits + return [(f["line"], f["col"], f["kind"], f["severity"]) for f in M.scan_text(text, path)] + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/workspace/scratch/7abea556a9be/auditor-origin-probe-20261109/tests/lean-subject_assertion-normal.py", line 712, in scan_text + raise AssertionError("ACTUAL_SUBJECT_ORIGIN_PROBE") +AssertionError: ACTUAL_SUBJECT_ORIGIN_PROBE + +---------------------------------------------------------------------- +Ran 1 test in 0.001s + +FAILED (failures=1) diff --git a/cbz/evidence/auditor-origin-20261109/lean-subject-optimized.stderr b/cbz/evidence/auditor-origin-20261109/lean-subject-optimized.stderr new file mode 100644 index 000000000..c3e6a7cf8 --- /dev/null +++ b/cbz/evidence/auditor-origin-20261109/lean-subject-optimized.stderr @@ -0,0 +1,20 @@ +test_native_decide_options (tests.test_cbz_layer_a_scan.ViolationTokens.test_native_decide_options) ... FAIL + +====================================================================== +FAIL: test_native_decide_options (tests.test_cbz_layer_a_scan.ViolationTokens.test_native_decide_options) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/workspace/scratch/7abea556a9be/federation-vector-post-integration-20261109/main/tests/test_cbz_layer_a_scan.py", line 64, in test_native_decide_options + self.assertEqual(hits("example : 2 = 2 := by decide +native"), [(1, 31, "native", "violation")]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/workspace/scratch/7abea556a9be/federation-vector-post-integration-20261109/main/tests/test_cbz_layer_a_scan.py", line 32, in hits + return [(f["line"], f["col"], f["kind"], f["severity"]) for f in M.scan_text(text, path)] + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/workspace/scratch/7abea556a9be/auditor-origin-probe-20261109/tests/lean-subject_assertion-optimized.py", line 712, in scan_text + raise AssertionError("ACTUAL_SUBJECT_ORIGIN_PROBE") +AssertionError: ACTUAL_SUBJECT_ORIGIN_PROBE + +---------------------------------------------------------------------- +Ran 1 test in 0.001s + +FAILED (failures=1) diff --git a/cbz/evidence/auditor-origin-20261109/main-lexical.json b/cbz/evidence/auditor-origin-20261109/main-lexical.json new file mode 100644 index 000000000..2090e7398 --- /dev/null +++ b/cbz/evidence/auditor-origin-20261109/main-lexical.json @@ -0,0 +1,36 @@ +{ + "summary": { + "files": 18, + "findings": 2, + "violations": 1, + "errors": 0, + "failing": 1, + "undecodable": [] + }, + "findings": [ + { + "path": "cbz/contributions/math514/kernel_replay/ExactReplay.lean", + "line": 13, + "col": 1, + "token": "unsafe", + "kind": "unsafe", + "layer": "A2", + "severity": "review", + "context": "production", + "context_path": "cbz/contributions/math514/kernel_replay/ExactReplay.lean", + "blob": "400f3853d50df7ea2a9b4bb2fc1de302626f2efa" + }, + { + "path": "cbz/contributions/math514/kernel_replay/tests/fixtures/UnapprovedAxiom.lean", + "line": 5, + "col": 8, + "token": "axiom", + "kind": "axiom", + "layer": "A2", + "severity": "violation", + "context": "production", + "context_path": "cbz/contributions/math514/kernel_replay/tests/fixtures/UnapprovedAxiom.lean", + "blob": "f802bd6affa61400830372158434e56cc1ae64b6" + } + ] +} \ No newline at end of file diff --git a/cbz/evidence/auditor-origin-20261109/math-a3.stdout b/cbz/evidence/auditor-origin-20261109/math-a3.stdout new file mode 100644 index 000000000..7a62d7b16 --- /dev/null +++ b/cbz/evidence/auditor-origin-20261109/math-a3.stdout @@ -0,0 +1,2 @@ +frontiers/source_derivative_gaussian_20261025/actual-source-finite-superlevel-components-evidence/raw/producer/history/freeze_documents.001.py: UNPARSEABLE SyntaxError: unmatched ')' (freeze_documents.001.py, line 37) +{"files": 1962, "by_class": {"FLOAT_SIGNAL": 1527, "MIXED": 395, "NO_FLOAT_SIGNAL": 39, "UNPARSEABLE": 1}} diff --git a/cbz/evidence/auditor-origin-20261109/math-lexical.json b/cbz/evidence/auditor-origin-20261109/math-lexical.json new file mode 100644 index 000000000..28e5fbbfd --- /dev/null +++ b/cbz/evidence/auditor-origin-20261109/math-lexical.json @@ -0,0 +1,11 @@ +{ + "summary": { + "files": 1974, + "findings": 0, + "violations": 0, + "errors": 0, + "failing": 0, + "undecodable": [] + }, + "findings": [] +} \ No newline at end of file diff --git a/cbz/evidence/auditor-origin-20261109/raw-failure-logs.zip b/cbz/evidence/auditor-origin-20261109/raw-failure-logs.zip new file mode 100644 index 000000000..0df81df26 Binary files /dev/null and b/cbz/evidence/auditor-origin-20261109/raw-failure-logs.zip differ diff --git a/cbz/evidence/auditor-origin-20261109/red-results-normal.json b/cbz/evidence/auditor-origin-20261109/red-results-normal.json new file mode 100644 index 000000000..c039e8da5 --- /dev/null +++ b/cbz/evidence/auditor-origin-20261109/red-results-normal.json @@ -0,0 +1,58 @@ +[ + { + "grader": "tests.test_cbz_layer_a_scan_mutants", + "mode": "normal", + "source": "tools/cbz/layer_a_scan.py", + "test_id": "tests.test_cbz_layer_a_scan.ViolationTokens.test_native_decide_options", + "kind": "baseline", + "subject_path": "/workspace/scratch/7abea556a9be/auditor-origin-probe-20261109/tests/lean-baseline-normal.py", + "original_sha256": "bf797f33d5df7a8e695f62e6a2d70dca05ec99956ac5c33aa0347164eac48410", + "subject_sha256": "bf797f33d5df7a8e695f62e6a2d70dca05ec99956ac5c33aa0347164eac48410", + "exit_code": 0, + "accepted_as_kill": false, + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stderr_sha256": "5eb908d5e37f5b7d76e3497b7462de88c68b1cd0648797c9690f6c67db66fd19" + }, + { + "grader": "tests.test_cbz_layer_a_scan_mutants", + "mode": "normal", + "source": "tools/cbz/layer_a_scan.py", + "test_id": "tests.test_cbz_layer_a_scan.ViolationTokens.test_native_decide_options", + "kind": "subject_assertion", + "subject_path": "/workspace/scratch/7abea556a9be/auditor-origin-probe-20261109/tests/lean-subject_assertion-normal.py", + "original_sha256": "bf797f33d5df7a8e695f62e6a2d70dca05ec99956ac5c33aa0347164eac48410", + "subject_sha256": "46130b771d8b29427eba1542fc8985b777e1aa697723f8c50170327363baf9e4", + "exit_code": 1, + "accepted_as_kill": true, + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stderr_sha256": "638a86cbfb8936b69b94b59b60ced3f225bc3186815b4aced643b72dc7673e24" + }, + { + "grader": "tests.test_cbz_a3_float_census", + "mode": "normal", + "source": "tools/cbz/a3_float_census.py", + "test_id": "tests.test_cbz_a3_float_census.Signals.test_float_and_complex_calls", + "kind": "baseline", + "subject_path": "/workspace/scratch/7abea556a9be/auditor-origin-probe-20261109/tests/a3-baseline-normal.py", + "original_sha256": "9faf7275053254e75372cd06d57de290b41124e86162c1c23eb5f4e46b6b2b80", + "subject_sha256": "9faf7275053254e75372cd06d57de290b41124e86162c1c23eb5f4e46b6b2b80", + "exit_code": 0, + "accepted_as_kill": false, + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stderr_sha256": "16d3b43b3c00e2a9da2bbb758497220d7ec22c1eb08e7a36065342bbcb482ac6" + }, + { + "grader": "tests.test_cbz_a3_float_census", + "mode": "normal", + "source": "tools/cbz/a3_float_census.py", + "test_id": "tests.test_cbz_a3_float_census.Signals.test_float_and_complex_calls", + "kind": "subject_assertion", + "subject_path": "/workspace/scratch/7abea556a9be/auditor-origin-probe-20261109/tests/a3-subject_assertion-normal.py", + "original_sha256": "9faf7275053254e75372cd06d57de290b41124e86162c1c23eb5f4e46b6b2b80", + "subject_sha256": "3a59127d1b0bedf08687ead6550a25222afcf5058756f4546b71f6accf0a2300", + "exit_code": 1, + "accepted_as_kill": true, + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stderr_sha256": "f1013343f9d1f9c9af5ac21be46b35691af3a960b1cbf066196235a8959b6c56" + } +] diff --git a/cbz/evidence/auditor-origin-20261109/red-results-optimized.json b/cbz/evidence/auditor-origin-20261109/red-results-optimized.json new file mode 100644 index 000000000..b0c766bbf --- /dev/null +++ b/cbz/evidence/auditor-origin-20261109/red-results-optimized.json @@ -0,0 +1,58 @@ +[ + { + "grader": "tests.test_cbz_layer_a_scan_mutants", + "mode": "optimized", + "source": "tools/cbz/layer_a_scan.py", + "test_id": "tests.test_cbz_layer_a_scan.ViolationTokens.test_native_decide_options", + "kind": "baseline", + "subject_path": "/workspace/scratch/7abea556a9be/auditor-origin-probe-20261109/tests/lean-baseline-optimized.py", + "original_sha256": "bf797f33d5df7a8e695f62e6a2d70dca05ec99956ac5c33aa0347164eac48410", + "subject_sha256": "bf797f33d5df7a8e695f62e6a2d70dca05ec99956ac5c33aa0347164eac48410", + "exit_code": 0, + "accepted_as_kill": false, + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stderr_sha256": "5eb908d5e37f5b7d76e3497b7462de88c68b1cd0648797c9690f6c67db66fd19" + }, + { + "grader": "tests.test_cbz_layer_a_scan_mutants", + "mode": "optimized", + "source": "tools/cbz/layer_a_scan.py", + "test_id": "tests.test_cbz_layer_a_scan.ViolationTokens.test_native_decide_options", + "kind": "subject_assertion", + "subject_path": "/workspace/scratch/7abea556a9be/auditor-origin-probe-20261109/tests/lean-subject_assertion-optimized.py", + "original_sha256": "bf797f33d5df7a8e695f62e6a2d70dca05ec99956ac5c33aa0347164eac48410", + "subject_sha256": "46130b771d8b29427eba1542fc8985b777e1aa697723f8c50170327363baf9e4", + "exit_code": 1, + "accepted_as_kill": true, + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stderr_sha256": "81a9f110c323e80b2fc6dc63252583ae3cb50b6bb5af766d7178df21b8167f84" + }, + { + "grader": "tests.test_cbz_a3_float_census", + "mode": "optimized", + "source": "tools/cbz/a3_float_census.py", + "test_id": "tests.test_cbz_a3_float_census.Signals.test_float_and_complex_calls", + "kind": "baseline", + "subject_path": "/workspace/scratch/7abea556a9be/auditor-origin-probe-20261109/tests/a3-baseline-optimized.py", + "original_sha256": "9faf7275053254e75372cd06d57de290b41124e86162c1c23eb5f4e46b6b2b80", + "subject_sha256": "9faf7275053254e75372cd06d57de290b41124e86162c1c23eb5f4e46b6b2b80", + "exit_code": 0, + "accepted_as_kill": false, + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stderr_sha256": "16d3b43b3c00e2a9da2bbb758497220d7ec22c1eb08e7a36065342bbcb482ac6" + }, + { + "grader": "tests.test_cbz_a3_float_census", + "mode": "optimized", + "source": "tools/cbz/a3_float_census.py", + "test_id": "tests.test_cbz_a3_float_census.Signals.test_float_and_complex_calls", + "kind": "subject_assertion", + "subject_path": "/workspace/scratch/7abea556a9be/auditor-origin-probe-20261109/tests/a3-subject_assertion-optimized.py", + "original_sha256": "9faf7275053254e75372cd06d57de290b41124e86162c1c23eb5f4e46b6b2b80", + "subject_sha256": "3a59127d1b0bedf08687ead6550a25222afcf5058756f4546b71f6accf0a2300", + "exit_code": 1, + "accepted_as_kill": true, + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stderr_sha256": "3d1718307c791d84185b176aca9147a81052fb41e0f30ebc9d4b6c7b49f4fbfd" + } +] diff --git a/cbz/evidence/auditor-origin-20261109/workspace-a3.stderr b/cbz/evidence/auditor-origin-20261109/workspace-a3.stderr new file mode 100644 index 000000000..4b29177af --- /dev/null +++ b/cbz/evidence/auditor-origin-20261109/workspace-a3.stderr @@ -0,0 +1 @@ +a3_float_census: error: gitlink requires a separately pinned scan: repos/Math- diff --git a/cbz/evidence/auditor-origin-20261109/workspace-lexical.stderr b/cbz/evidence/auditor-origin-20261109/workspace-lexical.stderr new file mode 100644 index 000000000..1262189fb --- /dev/null +++ b/cbz/evidence/auditor-origin-20261109/workspace-lexical.stderr @@ -0,0 +1 @@ +layer_a_scan: error: gitlink requires a separately pinned scan: repos/Math- diff --git a/tests/cbz_auditor_origin.py b/tests/cbz_auditor_origin.py new file mode 100644 index 000000000..6c6031975 --- /dev/null +++ b/tests/cbz_auditor_origin.py @@ -0,0 +1,103 @@ +"""Selected synchronous auditor mutation outcomes with live origin evidence. + +The existing inventory recorder captures traceback code identities before +unittest cleanup. Only the selected test body and its finite set of standard +unittest assertion helpers authorize a kill. This is a trusted test harness, +not a sandbox for hostile test code or a mathematical certifier. +""" +import hashlib +import json +import os +from pathlib import Path +import secrets +import sys + +from tests import cbz_inventory_origin + +ROOT = Path(__file__).resolve().parents[1] + + +def retain_full(tool_path, environment_key, process, before_bytes): + """Preserve full-suite baseline/failed-child logs when evidence is requested.""" + after_bytes = Path(tool_path).read_bytes() + directory = os.environ.get('CBZ_AUDITOR_EVIDENCE_DIR') + if directory: + target = Path(directory) / 'full' + target.mkdir(exist_ok=True) + nonce = secrets.token_hex(16) + stdout, stderr = process.stdout.encode(), process.stderr.encode() + for suffix, data in (('.before.py', before_bytes), ('.after.py', after_bytes), + ('.stdout', stdout), ('.stderr', stderr)): + with (target / (nonce + suffix)).open('xb') as stream: + stream.write(data) + row = {'nonce': nonce, 'environment_key': environment_key, + 'optimize': sys.flags.optimize, 'command': process.args, + 'exit_code': process.returncode, + 'source_before': hashlib.sha256(before_bytes).hexdigest(), + 'source_after': hashlib.sha256(after_bytes).hexdigest(), + 'stdout_sha256': hashlib.sha256(stdout).hexdigest(), + 'stderr_sha256': hashlib.sha256(stderr).hexdigest()} + with (target / (nonce + '.json')).open('x', encoding='utf-8') as stream: + json.dump(row, stream, sort_keys=True, indent=2) + stream.write('\n') + if before_bytes != after_bytes: + raise RuntimeError('full-suite subject changed during execution') + + +def run_selected(tool_path, environment_key, test_id): + """Run one designated test and bind the report to unchanged subject bytes.""" + try: + before_bytes = Path(tool_path).read_bytes() + before = hashlib.sha256(before_bytes).hexdigest() + except OSError as error: + return {'classification': 'invalid', 'protocol_error': True, 'error': repr(error)} + env = dict(os.environ, PYTHONDONTWRITEBYTECODE='1') + env[environment_key] = tool_path + report = cbz_inventory_origin.run(ROOT, test_id, [test_id], test_id, + env=env, timeout=600) + try: + after_bytes = Path(tool_path).read_bytes() + after = hashlib.sha256(after_bytes).hexdigest() + except OSError as error: + after_bytes, after = None, None + report.update(classification='invalid', protocol_error=True, error=repr(error)) + report['source_before'] = before + report['source_after'] = after + if before != after: + report['classification'] = 'invalid' + directory = os.environ.get('CBZ_AUDITOR_EVIDENCE_DIR') + if directory: + # An explicitly requested evidence write fails the test if it cannot + # be completed; collision-safe exclusive creation never overwrites a run. + nonce = secrets.token_hex(16) + report.update(evidence_nonce=nonce, environment_key=environment_key, + optimize=sys.flags.optimize) + target = Path(directory) + with (target / (nonce + '.before.py')).open('xb') as stream: + stream.write(before_bytes) + if after_bytes is not None: + with (target / (nonce + '.after.py')).open('xb') as stream: + stream.write(after_bytes) + with (target / (nonce + '.json')).open('x', encoding='utf-8') as stream: + json.dump(report, stream, sort_keys=True, indent=2) + stream.write('\n') + return report + + +def outcome(report, expected_test): + """Reclassify a source-bound report; refuse missing provenance or drift.""" + if type(report) is not dict or type(expected_test) is not str: + return 'invalid' + selected = report.get('selected_id') + before, after = report.get('source_before'), report.get('source_after') + if (type(selected) is not str or selected.rsplit('.', 1)[-1] != expected_test or + type(before) is not str or len(before) != 64 or + any(c not in '0123456789abcdef' for c in before) or before != after): + return 'invalid' + actual = cbz_inventory_origin.classify(report, [selected], selected, {}) + return actual if report.get('classification') == actual else 'invalid' + + +def killed(report, expected_test=None): + """Refuse anything except an authorized selected-body assertion failure.""" + return outcome(report, expected_test) == 'kill' diff --git a/tests/cbz_auditor_origin_mutation_child.py b/tests/cbz_auditor_origin_mutation_child.py new file mode 100644 index 000000000..f0ce34b37 --- /dev/null +++ b/tests/cbz_auditor_origin_mutation_child.py @@ -0,0 +1,35 @@ +"""Run one auditor-origin control with an explicitly selected helper copy.""" +import importlib.util +import json +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from tests import cbz_inventory_origin as recorder +from tests import test_cbz_auditor_origin as controls +from tests import test_cbz_a3_float_census as a3 +from tests import test_cbz_layer_a_scan_mutants as lean + + +def main(): + if len(sys.argv) != 3: + raise SystemExit('helper copy and exact selected test are required') + spec = importlib.util.spec_from_file_location('auditor_origin_mutant', sys.argv[1]) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + # The copied adapter lives outside the checkout; its tests still run + # against the exact checkout that owns the designated test bodies. + module.ROOT = ROOT + controls.cbz_auditor_origin = module + a3.cbz_auditor_origin = module + lean.cbz_auditor_origin = module + report = recorder._child(ROOT, sys.argv[2]) + print(json.dumps(report, sort_keys=True)) + return 0 if (report['failure_count'] == 0 and report['error_count'] == 0 and + report['unexpected_success_count'] == 0) else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/test_cbz_a3_float_census.py b/tests/test_cbz_a3_float_census.py index b7b56615f..7cb6e56b8 100644 --- a/tests/test_cbz_a3_float_census.py +++ b/tests/test_cbz_a3_float_census.py @@ -14,6 +14,8 @@ import tempfile import unittest +from tests import cbz_auditor_origin + HERE = os.path.dirname(os.path.abspath(__file__)) WORKSPACE = os.path.dirname(HERE) REAL = os.path.join(WORKSPACE, "tools", "cbz", "a3_float_census.py") @@ -213,44 +215,20 @@ def designated_test_ids(modules=TEST_MODULES): def run_suite(tool_path, test_id=None): + if test_id is not None: + return cbz_auditor_origin.run_selected(tool_path, 'CBZ_A3_CENSUS', test_id) + with open(tool_path, 'rb') as stream: + before = stream.read() env = dict(os.environ, CBZ_A3_CENSUS=tool_path, PYTHONDONTWRITEBYTECODE="1") - return subprocess.run([sys.executable, *(["-O"] if sys.flags.optimize else []), "-B", "-S", "-m", "unittest", "-v", + result = subprocess.run([sys.executable, *(["-O"] if sys.flags.optimize else []), "-B", "-S", "-m", "unittest", "-v", *([test_id] if test_id else TEST_MODULES)], cwd=WORKSPACE, env=env, capture_output=True, text=True, timeout=600) + cbz_auditor_origin.retain_full(tool_path, 'CBZ_A3_CENSUS', result, before) + return result -def killed(r, expected_test=None): - """A kill is exit 1 from the designated test run alone, failing only by its own assertions. - - Launch/import/syntax failures, errors, skips, zero or extra tests, foreign - failures and failures raised outside the designated method are not kills. - Nor is an AssertionError raised inside the tool under test: the innermost - traceback frame must be test code, so a crash counts the same whatever its - exception type. - """ - out = r.stdout + r.stderr - if expected_test is None or r.returncode != 1: - return False - runs = re.findall(r"^Ran [^\n]*$", out, re.M) - summary = re.findall(r"^FAILED \(failures=([1-9][0-9]*)\)$", out, re.M) - if (len(runs) != 1 or not re.fullmatch(r"Ran 1 test in [0-9]+(?:\.[0-9]+)?s", runs[0]) - or len(summary) != 1 or re.search(r"^FAILED \((?!failures=[1-9][0-9]*\)$)", out, re.M)): - return False - failures = 0 - for block in re.split(r"(?m)^={70}$", out)[1:]: - header = re.match(r"\n(FAIL|ERROR): (test_[A-Za-z0-9_]+) \(", block) - if header is None: - continue - if (header.group(1) != "FAIL" or header.group(2) != expected_test - or not re.search(r'^ File "[^"\n]+", line [0-9]+, in %s$' % re.escape(expected_test), block, re.M) - or not re.search(r"^AssertionError\b", block, re.M)): - return False - frames = re.findall(r'^ File "([^"\n]+)", line [0-9]+, in ', block, re.M) - if not frames or not re.search(r"[/\\]tests[/\\][^/\\]+\.py$", frames[-1]): - return False - failures += 1 - return failures == int(summary[0]) - +def killed(report, expected_test=None): + return cbz_auditor_origin.killed(report, expected_test) @unittest.skipIf(os.environ.get("CBZ_A3_CENSUS"), "mutant run: do not recurse") class SelfMutation(unittest.TestCase): @@ -261,6 +239,16 @@ def test_harness_control_unmutated_copy_passes(self): shutil.copyfile(REAL, path) r = run_suite(path) self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + skipped = [line for line in (r.stdout + r.stderr).splitlines() if ' ... skipped ' in line] + names = ('test_actual_syntax_broken_tool_is_not_a_kill', + 'test_nonbehavioral_failures_do_not_count_as_kills') + expected = [f"{name} (tests.test_cbz_auditor_hardening.MutationKillClassification.{name}) ... " + "skipped 'mutation subprocess: harness controls run only at top level'" for name in names] + names = ('test_every_mutant_is_killed', 'test_harness_control_unmutated_copy_passes') + expected += [f"{name} (tests.test_cbz_a3_float_census.SelfMutation.{name}) ... " + "skipped 'mutant run: do not recurse'" for name in names] + self.assertCountEqual(skipped, expected) + self.assertIn('OK (skipped=4)', r.stdout + r.stderr) def test_every_mutant_is_killed(self): with open(REAL, encoding="utf-8") as fh: @@ -270,10 +258,14 @@ def test_every_mutant_is_killed(self): with tempfile.TemporaryDirectory() as d: for i, (name, old, new) in enumerate(MUTANTS): self.assertEqual(src.count(old), 1, f"mutation point drifted: {name}") + test_id = ids[EXPECTED_FAILURE[name]] + baseline = run_suite(REAL, test_id) + self.assertEqual(cbz_auditor_origin.outcome(baseline, EXPECTED_FAILURE[name]), + 'baseline', (name, baseline)) path = os.path.join(d, f"m{i:02d}.py") with open(path, "w", encoding="utf-8") as fh: fh.write(src.replace(old, new)) - if not killed(run_suite(path, ids[EXPECTED_FAILURE[name]]), EXPECTED_FAILURE[name]): + if not killed(run_suite(path, test_id), EXPECTED_FAILURE[name]): survivors.append(name) self.assertEqual(survivors, []) diff --git a/tests/test_cbz_auditor_hardening.py b/tests/test_cbz_auditor_hardening.py index cdc1bb69f..d3e31089f 100644 --- a/tests/test_cbz_auditor_hardening.py +++ b/tests/test_cbz_auditor_hardening.py @@ -234,7 +234,8 @@ def test_nonbehavioral_failures_do_not_count_as_kills(self): with self.subTest(module=mod.__name__,out=out): self.assertFalse(mod.killed(subprocess.CompletedProcess([],1,stdout='',stderr=out),'test_expected')) good=fail_block('test_expected')+'-'*70+'\nRan 1 test in 0.001s\n\nFAILED (failures=1)\n' - self.assertTrue(mod.killed(subprocess.CompletedProcess([],1,stdout='',stderr=good),'test_expected')) + # Formatted text alone has no live traceback/code provenance. + self.assertFalse(mod.killed(subprocess.CompletedProcess([],1,stdout='',stderr=good),'test_expected')) a8=[(2,good), # wrong nonzero exit (1,fail_block('test_expected')+fail_block('test_other')+'-'*70+'\nRan 1 test in 0.001s\n\nFAILED (failures=2)\n'), (1,good.replace('Ran 1 test','Ran 3 tests')), # the designated test must run alone diff --git a/tests/test_cbz_auditor_origin.py b/tests/test_cbz_auditor_origin.py new file mode 100644 index 000000000..60c64b7f9 --- /dev/null +++ b/tests/test_cbz_auditor_origin.py @@ -0,0 +1,165 @@ +"""Real child-process origin controls for the Lean and Python auditor graders. + +Only source copies in a temporary directory are modified. The selected body, +subject and standard unittest assertion helpers remain distinguishable by the +live traceback recorder; filenames and exception text are not authority. +""" +import copy +import hashlib +import importlib +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest import mock + +from tests import cbz_auditor_origin + +ROOT = Path(__file__).resolve().parents[1] +CASES = ( + ('tests.test_cbz_layer_a_scan_mutants', 'tools/cbz/layer_a_scan.py', + 'test_native_decide_options', 'scan_text', 'return []'), + ('tests.test_cbz_a3_float_census', 'tools/cbz/a3_float_census.py', + 'test_float_and_complex_calls', 'census_text', + "return {'class': 'NO_FLOAT_SIGNAL', 'signals': []}"), +) + + +class AuditorOriginControls(unittest.TestCase): + def test_real_subject_assertions_cannot_count_as_oracle_failures(self): + for module, source, name, function, _ in CASES: + grader = importlib.import_module(module) + test_id = grader.designated_test_ids()[name] + with self.subTest(grader=module), tempfile.TemporaryDirectory() as tmp: + tests_dir = Path(tmp) / 'tests' + tests_dir.mkdir() + copy = tests_dir / 'subject.py' + copy.write_bytes((ROOT / source).read_bytes() + + ('\n\ndef ' + function + '(*args, **kwargs):\n' + ' raise AssertionError("SUBJECT_ORIGIN")\n').encode()) + result = grader.run_suite(str(copy), test_id) + self.assertFalse(grader.killed(result, name), result) + + def test_baselines_and_actual_selected_body_oracles(self): + for module, source, name, function, replacement in CASES: + grader = importlib.import_module(module) + test_id = grader.designated_test_ids()[name] + with self.subTest(grader=module), tempfile.TemporaryDirectory() as tmp: + tests_dir = Path(tmp) / 'tests' + tests_dir.mkdir() + copy = tests_dir / 'subject.py' + raw = (ROOT / source).read_bytes() + copy.write_bytes(raw) + baseline = grader.run_suite(str(copy), test_id) + self.assertEqual(cbz_auditor_origin.outcome(baseline, name), 'baseline', baseline) + self.assertFalse(grader.killed(baseline, name)) + copy.write_bytes(raw + ('\n\ndef ' + function + '(*args, **kwargs):\n' + ' ' + replacement + '\n').encode()) + self.assertTrue(grader.killed(grader.run_suite(str(copy), test_id), name)) + + def test_nested_subject_and_changed_source_are_refused(self): + for module, source, name, function, replacement in CASES: + grader = importlib.import_module(module) + test_id = grader.designated_test_ids()[name] + with self.subTest(grader=module), tempfile.TemporaryDirectory() as tmp: + tests_dir = Path(tmp) / 'tests' + tests_dir.mkdir() + subject = tests_dir / 'subject.py' + raw = (ROOT / source).read_bytes() + nested = ('\n\ndef ' + function + '(*args, **kwargs):\n' + ' def callback():\n' + ' raise AssertionError("NESTED_SUBJECT")\n' + ' return callback()\n') + subject.write_bytes(raw + nested.encode()) + report = grader.run_suite(str(subject), test_id) + self.assertFalse(grader.killed(report, name), report) + changed = ('\n\ndef ' + function + '(*args, **kwargs):\n' + ' with open(__file__, "ab") as f:\n' + ' f.write(b"\\n# source changed during child\\n")\n' + ' ' + replacement + '\n') + subject.write_bytes(raw + changed.encode()) + report = grader.run_suite(str(subject), test_id) + self.assertNotEqual(report.get('source_before'), report.get('source_after')) + self.assertFalse(grader.killed(report, name), report) + + def test_evidence_is_source_bound_and_write_failures_are_not_hidden(self): + module, source, name, function, replacement = CASES[0] + grader = importlib.import_module(module) + test_id = grader.designated_test_ids()[name] + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + subject = root / 'subject.py' + subject.write_bytes((ROOT / source).read_bytes() + + ('\n\ndef ' + function + '(*args, **kwargs):\n ' + replacement + '\n').encode()) + evidence = root / 'evidence' + evidence.mkdir() + with mock.patch.dict(os.environ, {'CBZ_AUDITOR_EVIDENCE_DIR': str(evidence)}): + report = grader.run_suite(str(subject), test_id) + self.assertTrue(grader.killed(report, name), report) + nonce = report['evidence_nonce'] + saved = json.loads((evidence / (nonce + '.json')).read_text()) + self.assertEqual(saved, report) + for suffix in ('.before.py', '.after.py'): + self.assertEqual(hashlib.sha256((evidence / (nonce + suffix)).read_bytes()).hexdigest(), + report['source_before']) + with mock.patch.dict(os.environ, {'CBZ_AUDITOR_EVIDENCE_DIR': str(root / 'missing')}): + with self.assertRaises(FileNotFoundError): + grader.run_suite(str(subject), test_id) + subject.write_bytes((ROOT / source).read_bytes()) + with mock.patch.dict(os.environ, {'CBZ_AUDITOR_EVIDENCE_DIR': str(evidence)}): + full = grader.run_suite(str(subject)) + self.assertEqual(full.returncode, 0, full.stderr) + records = list((evidence / 'full').glob('*.json')) + self.assertEqual(len(records), 1) + full_record = json.loads(records[0].read_text()) + self.assertEqual(full_record['exit_code'], full.returncode) + self.assertEqual(full_record['stderr_sha256'], hashlib.sha256(full.stderr.encode()).hexdigest()) + self.assertEqual((evidence / 'full' / (full_record['nonce'] + '.stderr')).read_text(), full.stderr) + + def test_missing_or_inconsistent_report_provenance_is_refused(self): + module, source, name, function, replacement = CASES[0] + grader = importlib.import_module(module) + test_id = grader.designated_test_ids()[name] + with tempfile.TemporaryDirectory() as tmp: + subject = Path(tmp) / 'subject.py' + subject.write_bytes((ROOT / source).read_bytes() + + ('\n\ndef ' + function + '(*args, **kwargs):\n ' + replacement + '\n').encode()) + good = grader.run_suite(str(subject), test_id) + self.assertTrue(grader.killed(good, name), good) + variants = [] + for field in ('source_before', 'source_after', 'selected_id', 'events'): + bad = copy.deepcopy(good) + bad.pop(field) + variants.append(bad) + bad = copy.deepcopy(good) + bad['source_after'] = '0' * 64 + variants.append(bad) + for fake in ('g' * 64, '0' * 63): + bad = copy.deepcopy(good) + bad['source_before'] = bad['source_after'] = fake + variants.append(bad) + bad = copy.deepcopy(good) + bad['events'][0]['authorized'] = False + variants.append(bad) + bad = copy.deepcopy(good) + bad['classification'] = 'baseline' + variants.append(bad) + for bad in variants: + with self.subTest(bad=bad): + self.assertFalse(grader.killed(bad, name)) + + def test_traceback_text_without_live_provenance_is_not_a_kill(self): + fake = subprocess.CompletedProcess([], 1, stdout='', stderr=( + '=' * 70 + '\nFAIL: test_expected (fixture.Test.test_expected)\n' + ' File "/tmp/tests/fake.py", line 1, in test_expected\n' + 'AssertionError: fake\n' + '-' * 70 + + '\nRan 1 test in 0.001s\n\nFAILED (failures=1)\n')) + for module, _, _, _, _ in CASES: + with self.subTest(grader=module): + self.assertFalse(importlib.import_module(module).killed(fake, 'test_expected')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_cbz_auditor_origin_mutations.py b/tests/test_cbz_auditor_origin_mutations.py new file mode 100644 index 000000000..530565c0e --- /dev/null +++ b/tests/test_cbz_auditor_origin_mutations.py @@ -0,0 +1,87 @@ +"""Adverse source mutations of the auditor-origin result adapter.""" +import hashlib +import json +import os +from pathlib import Path +import secrets +import subprocess +import sys +import tempfile +import unittest + +from tests import cbz_inventory_origin as recorder + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / 'tests' / 'cbz_auditor_origin.py' +CHILD = ROOT / 'tests' / 'cbz_auditor_origin_mutation_child.py' +PREFIX = 'tests.test_cbz_auditor_origin.AuditorOriginControls.' +MUTANTS = ( + ('accept any outcome', "return outcome(report, expected_test) == 'kill'", 'return True', + 'test_baselines_and_actual_selected_body_oracles'), + ('source drift accepted', 'or before != after):', 'or False):', + 'test_missing_or_inconsistent_report_provenance_is_refused'), + ('nonhex source digest accepted', "any(c not in '0123456789abcdef' for c in before)", 'False', + 'test_missing_or_inconsistent_report_provenance_is_refused'), + ('wrong digest length accepted', 'len(before) != 64', 'False', + 'test_missing_or_inconsistent_report_provenance_is_refused'), + ('inconsistent classification accepted', "report.get('classification') == actual", 'True', + 'test_missing_or_inconsistent_report_provenance_is_refused'), +) + + +def run(path, target, label): + before = Path(path).read_bytes() + env = dict(os.environ) + env.pop('CBZ_AUDITOR_EVIDENCE_DIR', None) + command = [sys.executable, '-B', *(['-O'] if sys.flags.optimize else []), '-S', + str(CHILD), str(path), target] + process = subprocess.run(command, cwd=ROOT, env=env, capture_output=True, timeout=180) + after = Path(path).read_bytes() + try: + report = json.loads(process.stdout) + report.update(protocol_error=False, child_exit=process.returncode, selected_id=target) + classification = recorder.classify(report, [target], target, {}) + except (ValueError, TypeError, AttributeError): + report, classification = {}, 'invalid' + if before != after: + classification = 'invalid' + directory = os.environ.get('CBZ_AUDITOR_ORIGIN_MUTATION_EVIDENCE_DIR') + if directory: + nonce = secrets.token_hex(16) + dest = Path(directory) + for suffix, data in (('.source.py', before), ('.stdout', process.stdout), + ('.stderr', process.stderr)): + with (dest / (nonce + suffix)).open('xb') as stream: + stream.write(data) + entry = {'nonce': nonce, 'label': label, 'target': target, + 'optimize': sys.flags.optimize, 'command': command, + 'exit_code': process.returncode, 'classification': classification, + 'source_before': hashlib.sha256(before).hexdigest(), + 'source_after': hashlib.sha256(after).hexdigest(), + 'stdout_sha256': hashlib.sha256(process.stdout).hexdigest(), + 'stderr_sha256': hashlib.sha256(process.stderr).hexdigest()} + with (dest / (nonce + '.json')).open('x', encoding='utf-8') as stream: + json.dump(entry, stream, indent=2, sort_keys=True) + stream.write('\n') + return classification, report, process.stderr.decode(errors='replace') + + +class AuditorOriginMutations(unittest.TestCase): + def test_actual_mutants_have_clean_baselines_and_authorized_failures(self): + source = SOURCE.read_text() + with tempfile.TemporaryDirectory() as directory: + subject = Path(directory) / 'origin_adapter.py' + for name, old, new, test in MUTANTS: + with self.subTest(mutant=name): + self.assertEqual(source.count(old), 1, name) + target = PREFIX + test + subject.write_text(source) + baseline = run(subject, target, name + ':baseline') + self.assertEqual(baseline[0], 'baseline', baseline) + subject.write_text(source.replace(old, new)) + mutant = run(subject, target, name + ':mutant') + self.assertEqual(mutant[0], 'kill', mutant) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_cbz_auditor_origin_workflow.py b/tests/test_cbz_auditor_origin_workflow.py new file mode 100644 index 000000000..3fe0ae3fe --- /dev/null +++ b/tests/test_cbz_auditor_origin_workflow.py @@ -0,0 +1,111 @@ +"""Exercise the actual workflow shell guards against synthetic child outcomes.""" +import os +from pathlib import Path +import subprocess +import tempfile +import textwrap +import unittest + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / '.github/workflows/cbz-auditor-origin.yml' + + +def workflow_block(): + source = WORKFLOW.read_text() + start = ' - name: Run both actual modes and retain each mutation observation\n' + end = ' - name: Preserve complete observations including failures\n' + if source.count(start) != 1 or source.count(end) != 1: + raise ValueError('ambiguous workflow step') + step = source.split(start, 1)[1].split(end, 1)[0] + if step.count(' run: |\n') != 1: + raise ValueError('missing workflow shell block') + return textwrap.dedent(step.split(' run: |\n', 1)[1]) + + +PYTHON_STUB = r'''#!/usr/bin/env python3 +import os, pathlib, sys +if '--version' in sys.argv: + print('Python synthetic guard fixture'); raise SystemExit(0) +origin = 'tests.test_cbz_auditor_origin_mutations' in sys.argv +key = 'CBZ_AUDITOR_ORIGIN_MUTATION_EVIDENCE_DIR' if origin else 'CBZ_AUDITOR_EVIDENCE_DIR' +count = 10 if origin else 202 +if os.environ.get('CBZ_GUARD_CASE') == 'missing' and not origin: + count -= 1 +out = pathlib.Path(os.environ[key]) +if os.environ.get('CBZ_GUARD_CASE') == 'origin_exit' and origin: + (out.parent / 'origin.exit').mkdir() +if os.environ.get('CBZ_GUARD_CASE') == 'census_exit' and not origin: + (out.parent / 'census.exit').mkdir() +for i in range(count): + (out / ('fixture-%03d.json' % i)).write_text('{}') +print('Ran %d tests in 0.001s' % (22 if origin else 83)) +if os.environ.get('CBZ_GUARD_CASE') == 'skip' and origin: + print("test_fixture ... skipped 'synthetic'") +print('OK') +raise SystemExit(1 if os.environ.get('CBZ_GUARD_CASE') == 'failure' and origin else 0) +''' +GIT_STUB = '''#!/bin/sh +case "$1" in + rev-parse) printf 'synthetic-source\\n' ;; + status) if [ "$CBZ_GUARD_CASE" = dirty ]; then printf ' M source\\n'; fi ;; + *) exit 2 ;; +esac +''' +HASH_STUB = '''#!/bin/sh +if [ "$1" = -c ] && [ "$CBZ_GUARD_CASE" = hash ]; then exit 1; fi +exec /usr/bin/sha256sum "$@" +''' + + +def run_block(block, variant): + with tempfile.TemporaryDirectory(prefix='cbz-origin-workflow-guard-') as tmp: + root = Path(tmp) + binary = root / 'bin' + binary.mkdir() + for name, data in [('python', PYTHON_STUB), ('git', GIT_STUB), ('sha256sum', HASH_STUB)]: + p = binary / name + p.write_text(data) + p.chmod(0o755) + script = root / 'workflow.sh' + script.write_text(block) + env = dict(os.environ, PATH=str(binary) + os.pathsep + os.defpath, + RUNNER_TEMP=str(root), CBZ_GUARD_CASE=variant) + return subprocess.run(['bash', str(script)], cwd=ROOT, env=env, + text=True, capture_output=True, timeout=30) + + +class AuditorOriginWorkflowGuards(unittest.TestCase): + def test_exit_status_write_failures_are_not_green(self): + block = workflow_block() + for variant in ('origin_exit', 'census_exit'): + with self.subTest(variant=variant): + result = run_block(block, variant) + self.assertNotEqual(result.returncode, 0, result.stderr) + + def test_actual_shell_guards_and_removals(self): + block = workflow_block() + baseline = run_block(block, 'good') + self.assertEqual(baseline.returncode, 0, baseline.stderr) + variants = ( + ('dirty', 'if [ -s "$evidence/worktree-before.txt" ]; then', 'if false; then'), + ('failure', 'else status=$?; failed=1; fi', 'else status=$?; :; fi'), + ('skip', "if grep -Fq ' ... skipped '", "if false && grep -Fq ' ... skipped '"), + ('missing', '-eq 202 ] || failed=1', '-eq 201 ] || failed=1'), + ('origin_exit', '> "$evidence/$mode/origin.exit" || failed=1', + '> "$evidence/$mode/origin.exit" || :'), + ('census_exit', '> "$evidence/$mode/census.exit" || failed=1', + '> "$evidence/$mode/census.exit" || :'), + ('hash', '"$evidence/source-sha256.txt" > "$evidence/source-readback.log" 2>&1 || failed=1', + '"$evidence/source-sha256.txt" > "$evidence/source-readback.log" 2>&1 || :'), + ) + for variant, old, new in variants: + with self.subTest(variant=variant): + self.assertEqual(block.count(old), 2 if variant == 'failure' else 1) + rejected = run_block(block, variant) + self.assertNotEqual(rejected.returncode, 0, variant) + removed = run_block(block.replace(old, new, 1), variant) + self.assertEqual(removed.returncode, 0, (variant, removed.stderr)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_cbz_layer_a_scan_mutants.py b/tests/test_cbz_layer_a_scan_mutants.py index faab4ff7e..564782fea 100644 --- a/tests/test_cbz_layer_a_scan_mutants.py +++ b/tests/test_cbz_layer_a_scan_mutants.py @@ -2,14 +2,12 @@ Each mutant is a copy of tools/cbz/layer_a_scan.py with one deliberate defect: a hazard token dropped, a lexer rule broken, or a fail-closed exit weakened. -The scanner's own suite (tests/test_cbz_layer_a_scan.py) is run against every -mutant through $CBZ_LAYER_A_SCAN and must FAIL. A surviving mutant means the -suite cannot tell a broken auditor from a sound one, so this test fails. +Its designated test must pass against the original and fail against the mutant. Every replacement must match the scanner source exactly once, so the harness -cannot silently drift out of sync with the code it mutates. A mutant counts as -killed only when the suite loaded and ran, and at least one test failed or -errored; a suite that could not even be imported proves nothing. +cannot silently drift out of sync with the code it mutates. A kill requires an +actual selected-body or explicitly authorized unittest assertion origin, with +source bytes unchanged across the child run. Errors and skips are not kills. """ import ast import os @@ -19,6 +17,8 @@ import tempfile import unittest +from tests import cbz_auditor_origin + HERE = os.path.dirname(os.path.abspath(__file__)) WORKSPACE = os.path.dirname(HERE) SCANNER = os.path.join(WORKSPACE, "tools", "cbz", "layer_a_scan.py") @@ -154,44 +154,20 @@ def designated_test_ids(modules=TEST_MODULES): def run_suite(tool_path, test_id=None): + if test_id is not None: + return cbz_auditor_origin.run_selected(tool_path, 'CBZ_LAYER_A_SCAN', test_id) + with open(tool_path, 'rb') as stream: + before = stream.read() env = dict(os.environ, CBZ_LAYER_A_SCAN=tool_path, PYTHONDONTWRITEBYTECODE="1") - return subprocess.run([sys.executable, *(["-O"] if sys.flags.optimize else []), "-B", "-S", "-m", "unittest", "-v", + result = subprocess.run([sys.executable, *(["-O"] if sys.flags.optimize else []), "-B", "-S", "-m", "unittest", "-v", *([test_id] if test_id else TEST_MODULES)], cwd=WORKSPACE, env=env, capture_output=True, text=True, timeout=600) + cbz_auditor_origin.retain_full(tool_path, 'CBZ_LAYER_A_SCAN', result, before) + return result -def killed(r, expected_test=None): - """A kill is exit 1 from the designated test run alone, failing only by its own assertions. - - Launch/import/syntax failures, errors, skips, zero or extra tests, foreign - failures and failures raised outside the designated method are not kills. - Nor is an AssertionError raised inside the tool under test: the innermost - traceback frame must be test code, so a crash counts the same whatever its - exception type. - """ - out = r.stdout + r.stderr - if expected_test is None or r.returncode != 1: - return False - runs = re.findall(r"^Ran [^\n]*$", out, re.M) - summary = re.findall(r"^FAILED \(failures=([1-9][0-9]*)\)$", out, re.M) - if (len(runs) != 1 or not re.fullmatch(r"Ran 1 test in [0-9]+(?:\.[0-9]+)?s", runs[0]) - or len(summary) != 1 or re.search(r"^FAILED \((?!failures=[1-9][0-9]*\)$)", out, re.M)): - return False - failures = 0 - for block in re.split(r"(?m)^={70}$", out)[1:]: - header = re.match(r"\n(FAIL|ERROR): (test_[A-Za-z0-9_]+) \(", block) - if header is None: - continue - if (header.group(1) != "FAIL" or header.group(2) != expected_test - or not re.search(r'^ File "[^"\n]+", line [0-9]+, in %s$' % re.escape(expected_test), block, re.M) - or not re.search(r"^AssertionError\b", block, re.M)): - return False - frames = re.findall(r'^ File "([^"\n]+)", line [0-9]+, in ', block, re.M) - if not frames or not re.search(r"[/\\]tests[/\\][^/\\]+\.py$", frames[-1]): - return False - failures += 1 - return failures == int(summary[0]) - +def killed(report, expected_test=None): + return cbz_auditor_origin.killed(report, expected_test) class ScannerMutants(unittest.TestCase): @classmethod @@ -202,6 +178,13 @@ def setUpClass(cls): def test_unmutated_scanner_passes_its_suite(self): r = run_suite(SCANNER) self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + skipped = [line for line in (r.stdout + r.stderr).splitlines() if ' ... skipped ' in line] + names = ('test_actual_syntax_broken_tool_is_not_a_kill', + 'test_nonbehavioral_failures_do_not_count_as_kills') + expected = [f"{name} (tests.test_cbz_auditor_hardening.MutationKillClassification.{name}) ... " + "skipped 'mutation subprocess: harness controls run only at top level'" for name in names] + self.assertCountEqual(skipped, expected) + self.assertIn('OK (skipped=2)', r.stdout + r.stderr) def test_every_mutant_is_killed(self): survivors = [] @@ -210,10 +193,14 @@ def test_every_mutant_is_killed(self): for i, (name, old, new) in enumerate(MUTANTS): with self.subTest(mutant=name): self.assertEqual(self.source.count(old), 1, f"mutation point drifted: {name}") + test_id = ids[EXPECTED_FAILURE[name]] + baseline = run_suite(SCANNER, test_id) + self.assertEqual(cbz_auditor_origin.outcome(baseline, EXPECTED_FAILURE[name]), + 'baseline', (name, baseline)) path = os.path.join(d, f"mutant_{i:02d}.py") with open(path, "w", encoding="utf-8") as fh: fh.write(self.source.replace(old, new)) - r = run_suite(path, ids[EXPECTED_FAILURE[name]]) + r = run_suite(path, test_id) if not killed(r, EXPECTED_FAILURE[name]): survivors.append(name) self.assertEqual(survivors, [], f"surviving mutants: {survivors}")