diff --git a/.github/workflows/cbz-g11-auditor.yml b/.github/workflows/cbz-g11-auditor.yml new file mode 100644 index 000000000..0d3fe0c64 --- /dev/null +++ b/.github/workflows/cbz-g11-auditor.yml @@ -0,0 +1,28 @@ +name: CBZ G11 auditor component self-tests + +on: + pull_request: + push: + +permissions: + contents: read + +jobs: + g11-auditor: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: '3.11' + - name: Run census controls normally + run: env -u PYTHONPATH -u PYTHONHOME -u PYTHONPYCACHEPREFIX -u PYTHONSTARTUP -u PYTHONUSERBASE python3 -B -S -m unittest tests.test_g11_census + - name: Run census controls with optimization + run: env -u PYTHONPATH -u PYTHONHOME -u PYTHONPYCACHEPREFIX -u PYTHONSTARTUP -u PYTHONUSERBASE python3 -B -S -O -m unittest tests.test_g11_census + - name: Run mutation self-test normally + run: env -u PYTHONPATH -u PYTHONHOME -u PYTHONPYCACHEPREFIX -u PYTHONSTARTUP -u PYTHONUSERBASE python3 -B -S tools/cbz/g11_mutation_selftest.py + - name: Run mutation self-test with optimization + run: env -u PYTHONPATH -u PYTHONHOME -u PYTHONPYCACHEPREFIX -u PYTHONSTARTUP -u PYTHONUSERBASE python3 -B -S -O tools/cbz/g11_mutation_selftest.py diff --git a/docs/cbz/G11_AUDITOR.md b/docs/cbz/G11_AUDITOR.md new file mode 100644 index 000000000..1d527e252 --- /dev/null +++ b/docs/cbz/G11_AUDITOR.md @@ -0,0 +1,30 @@ +# G11 auditor — bounded census, not certification + +## Purpose + +`tools/cbz/g11_census.py` reads tracked blob bytes from one Git revision and reports review triggers for the owner-defined G11 tooling-hygiene criterion: Python stubs, code-comment markers, no-op workflow run steps, and tool documentation/static-reference checks. It is a triage instrument, not a proof, a Layer A certificate, or a federation-wide G11 admission. + +## Commands + +From the repository root, using only the Python standard library: + +```sh +python3 -B -S -m unittest tests.test_g11_census +python3 -B -S -O -m unittest tests.test_g11_census +python3 -B -S tools/cbz/g11_mutation_selftest.py +python3 -B -S -O tools/cbz/g11_mutation_selftest.py +python3 -B -S tools/cbz/g11_census.py --rev --json census.json +``` + +The census exits 0 for no detected production finding, 1 for a detected production finding, and 2 when the requested Git revision cannot be read. In-scope tracked symlinks and other non-regular scanned entries fail closed with exit 2. A zero exit is only the result of this implementation over its stated scanned paths and rules. + +## Soundness boundaries + +- Stub exemptions are bound to source-resolved imports, not decorator or base names. An imported `abc.abstractmethod` exempts only a method in an immediate class with a trusted imported `abc.ABC` base. Imported `typing.overload`/`typing_extensions.overload` exempts only declarations in one contiguous same-name overload group followed by a concrete, non-stub implementation. Immediate imported `typing.Protocol`/`typing_extensions.Protocol` interface methods and the named `http.server` request-handler `log_message` override remain contextual. ABC inheritance alone and exception-class inheritance do not exempt a method. Nested outer-class context, unpaired overloads, name suffixes, local look-alikes, and lexical bindings shadowing an imported decorator/base do not grant exemptions. +- Python reference evidence is limited to module-level statements and classes whose base resolves to the imported `unittest.TestCase` identity (including explicit aliases), following only locally reachable helper calls and literal branches; a local class merely named `TestCase` is not credited. Uncalled helpers, `if False`, runtime-dependent branches, and loop bodies are not credited. Workflow references ignore here-document bodies and are read from run commands, not YAML comments. Python/shell `-c` payloads and JavaScript template-literal text are not credited. These are static references only, never proof of runtime execution, and cannot satisfy a runtime-exercise requirement. +- YAML comments are scanned for marker words. Empty and comment-only `run` blocks are reported as no-op steps. +- The mutation self-test has 75 cases: 72 of the 74 original cases, one F4 decoy-`TestCase` control, and two controls for the Python 3.11+ unittest failure header `method (module.Class.method)`, with and without a `subTest` suffix. Two original cases were dropped as equivalent mutants: exception bases confer no stub exemption, so neither a decoy `…Error` base name nor an ignored shadow of a builtin exception name can change a report. It seals the exact source slice containing `MUTATIONS`, `TARGETS`, `NEW_MUTATIONS` and `EXPECTED_MUTATION_NAMES` with a test-side SHA-256, then compares complete literal case tuples—not names alone—against the harness source. A kill is admitted only when the clean full 38-test baseline succeeds and the exact designated module/class/method reports `FAILED` with one AssertionError at its exact designated assertion line, zero errors, and zero skips. The harness propagates the parent's optimized mode to child tests and applies each mutation to its declared source, including all harness-target cases. + +## Limits + +This scanner does not establish Python runtime behavior, shell/YAML semantics in general, JavaScript parsing correctness, completeness of every possible test/execution mechanism, or semantic absence of placeholders introduced by metaprogramming. The comment lexer for JavaScript/shell is intentionally conservative. Findings and misses require source-aware review. A green component workflow is not G11 gate acceptance, Layer A compliance, proof closure, scientific acceptance, or CBZ-0. diff --git a/tests/test_g11_census.py b/tests/test_g11_census.py new file mode 100644 index 000000000..e025ed88d --- /dev/null +++ b/tests/test_g11_census.py @@ -0,0 +1,605 @@ +"""Tests for the CBZ-0 G11 census (tools/cbz/g11_census.py). + +Every finding class is planted once in a synthetic tree, beside look-alikes that +must not be flagged; the mutation self-test (tools/cbz/g11_mutation_selftest.py) +requires these tests to fail when any detector is weakened. +""" +import hashlib +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools" / "cbz")) +import g11_census as g11 # noqa: E402 +import g11_mutation_selftest as g11m # noqa: E402 + +EXPECTED_MUTATION_INVENTORY_SHA256 = "333f981944fbd08389c9fb384d6cd0dbf89fff4e40514ca67b123a0057be5d77" + +DOCUMENTED_TESTED = { + "tools/README.md": b"Run `tools/good.py` to check things.\n", + "tests/test_good.py": b"import good\n", +} + + +def run(tree, all_paths=None): + return g11.census({k: (v.encode() if isinstance(v, str) else v) for k, v in tree.items()}, all_paths) + + +def classes(report, path=None, area=None): + return sorted((f["class"], f["line"]) for f in report["findings"] + if (path is None or f["path"] == path) and (area is None or f["area"] == area)) + + +def stubs(text, path="tools/good.py"): + return [(f["line"], f["detail"]) for f in run({path: text})["findings"] + if f["class"] in ("stub", "stub-context")] + + +class Areas(unittest.TestCase): + def test_area_of_each_path_family(self): + cases = {"tools/a.py": "tooling", ".github/workflows/x.yml": "tooling", + "docs/site/app.mjs": "site", "docs/site/vendor/three.js": "vendor", + "tests/test_a.py": "tests", "tools/x/test_y.py": "tests", "conftest.py": "tests", + "docs/site/app.test.mjs": "tests", "experiments/e.py": "research", + "reviews/r/s.py": "evidence", "docs/reports/h.py": "evidence"} + for path, expected in cases.items(): + with self.subTest(path=path): + self.assertEqual(g11.area(path), expected) + + +class PythonStubs(unittest.TestCase): + def test_each_stub_body(self): + self.assertEqual(stubs("def a():\n pass\n"), [(1, "a: pass")]) + self.assertEqual(stubs("def a():\n ...\n"), [(1, "a: ellipsis")]) + self.assertEqual(stubs("def a():\n raise NotImplementedError\n"), [(1, "a: raise NotImplementedError")]) + self.assertEqual(stubs("def a():\n raise NotImplementedError('later')\n"), + [(1, "a: raise NotImplementedError")]) + self.assertEqual(stubs('def a():\n """Only a docstring."""\n'), [(1, "a: docstring only")]) + + def test_docstring_then_stub_body(self): + self.assertEqual(stubs('def a():\n """Doc."""\n pass\n'), [(1, "a: pass")]) + + def test_methods_async_and_nested(self): + text = ("class K:\n def m(self):\n pass\n" + "async def b():\n ...\n" + "def outer():\n def inner():\n pass\n return inner\n") + self.assertEqual(stubs(text), [(2, "m: pass"), (4, "b: ellipsis"), (7, "inner: pass")]) + + def test_real_bodies_are_not_stubs(self): + for text in ("def a():\n return 1\n", + 'def a():\n """Doc."""\n return 1\n', + "def a(x):\n if x:\n pass\n return x\n", + "def a():\n pass\n return 2\n", + "def a():\n raise ValueError('bad')\n", + "class E(Exception):\n pass\n"): + with self.subTest(text=text): + self.assertEqual(stubs(text), []) + + def test_legitimate_contexts_are_reported_but_do_not_fail(self): + text = ("from abc import ABC, abstractmethod\nfrom typing import Protocol, overload\n" + "class Base(ABC):\n def m(self):\n ...\n" + "class Shape(Protocol):\n def area(self) -> float:\n ...\n" + "class Bad(ValueError):\n def __init__(self):\n pass\n" + "class C(ABC):\n @abstractmethod\n def a(self):\n pass\n" + " @overload\n def f(self, x: int) -> int:\n ...\n" + " @overload\n def f(self, x: str) -> str:\n ...\n" + " def f(self, x):\n return x\n") + report = run({"tools/good.py": text, **DOCUMENTED_TESTED}) + contexts = sorted(f["detail"] for f in report["findings"] if f["class"] == "stub-context") + self.assertEqual(contexts, ["a: pass; @abstractmethod", + "area: ellipsis; method of a Protocol class", + "f: ellipsis; @overload", "f: ellipsis; @overload"]) + self.assertEqual(sorted(f["detail"] for f in report["failing"]), ["__init__: pass", "m: ellipsis"]) + self.assertEqual(report["status"], "FAIL") + + def test_request_handler_log_message_override_is_legitimate(self): + text = ("from http.server import SimpleHTTPRequestHandler\n" + "class Handler(SimpleHTTPRequestHandler):\n def log_message(self, *args):\n pass\n" + " def do_GET(self):\n pass\n" + "class Other:\n def log_message(self, *args):\n pass\n") + found = sorted((f["class"], f["line"]) for f in run({"tools/good.py": text})["findings"] + if f["class"].startswith("stub")) + self.assertEqual(found, [("stub", 5), ("stub", 8), ("stub-context", 3)]) + + def test_production_stub_fails_but_research_stub_does_not(self): + bad = run({"tools/good.py": "def a():\n pass\n", **DOCUMENTED_TESTED}) + self.assertEqual([(f["path"], f["class"]) for f in bad["failing"]], [("tools/good.py", "stub")]) + self.assertEqual(bad["status"], "FAIL") + research = run({"experiments/e.py": "def a():\n pass\n"}) + self.assertEqual(classes(research), [("stub", 1)]) + self.assertEqual(research["status"], "PASS") + + +class Markers(unittest.TestCase): + def test_python_comment_is_marker_and_string_is_mention(self): + text = ('def a():\n # TODO: finish\n return "TODO in data"\n' + '# FIXME later\nx = 1 # XXX\n# TBD\n# STUB\n# not implemented yet\n') + self.assertEqual(classes(run({"tools/good.py": text}), "tools/good.py"), + [("marker", 2), ("marker", 4), ("marker", 5), ("marker", 6), ("marker", 7), + ("marker", 8), ("mention", 3)]) + + def test_lowercase_words_and_lookalikes_are_not_markers(self): + text = "# todo list, xxx-large, stubs, tbd-free\nTODOS = 1 # TODOS are fine\n" + self.assertEqual(classes(run({"tools/good.py": text}), "tools/good.py"), []) + + def test_docstring_marker_is_a_mention(self): + self.assertEqual(classes(run({"tools/good.py": 'def a():\n """TODO: doc."""\n return 1\n'})), + [("mention", 2)]) + + def test_javascript_comments_and_strings(self): + text = ('// TODO top\nconst s = "TODO not a comment";\nconst u = "http://x"; // FIXME tail\n' + "/* XXX\n still TBD */ const t = 'STUB';\nconst r = `// TODO in template`;\n") + self.assertEqual(classes(run({"docs/site/app.mjs": text})), + [("marker", 1), ("marker", 3), ("marker", 4), ("marker", 5), + ("mention", 2), ("mention", 5), ("mention", 6)]) + + def test_escaped_quote_inside_string(self): + text = 'const s = "a \\" // TODO still data";\n' + self.assertEqual(classes(run({"docs/site/app.mjs": text})), [("mention", 1)]) + + def test_shell_comments_and_strings(self): + text = '#!/bin/sh\n# TODO setup\necho "TODO in data" # FIXME\nn=${#TODO}\n' + self.assertEqual(classes(run({"tools/good.sh": text})), + [("marker", 2), ("marker", 3), ("mention", 3), ("mention", 4)]) + + def test_production_marker_fails(self): + report = run({"tools/good.py": "# TODO\n", **DOCUMENTED_TESTED}) + self.assertEqual(report["status"], "FAIL") + + def test_site_marker_fails_and_vendor_marker_does_not(self): + self.assertEqual(run({"docs/site/app.mjs": "// FIXME\n"})["status"], "FAIL") + self.assertEqual(run({"docs/site/vendor/lib.js": "// FIXME\n"})["status"], "PASS") + + +class Workflows(unittest.TestCase): + def test_noop_steps(self): + text = ("jobs:\n a:\n steps:\n" + " - run: true\n" + " - name: colon\n run: ':'\n" + " - run: exit 0 # placeholder\n" + " - name: block\n run: |\n # nothing\n true\n shell: bash\n" + " - run: |\n true\n shell: bash\n" + " - run: echo hi\n" + " - name: real block\n run: |\n true\n make test\n") + found = [f["line"] for f in run({".github/workflows/w.yml": text})["findings"] if f["class"] == "noop"] + self.assertEqual(found, [4, 6, 7, 9, 13]) + + def test_production_noop_fails(self): + self.assertEqual(run({".github/workflows/w.yml": " - run: true\n"})["status"], "FAIL") + self.assertEqual(run({".github/workflows/w.yml": " - run: make\n"})["status"], "PASS") + + +class ToolChecks(unittest.TestCase): + def tools(self, tree, all_paths=None): + return run(tree, all_paths)["tools"] + + def test_documented_and_exercised_by_path_import_and_basename(self): + tree = {"tools/a.py": "x = 1\n", "tools/b.py": "x = 1\n", "tools/c.mjs": "export const x = 1;\n", "tools/d.py": "x = 1\n", + "docs/USE.md": "See tools/a.py, tools/b.py, tools/c.mjs and d.py.\n", + "tests/test_a.py": "from tools import helper_x, a\n", + "tests/test_d.py": "import d\n", + ".github/workflows/ci.yml": " - run: python tools/b.py\n", + "tests/c.test.mjs": "import { x } from '../tools/c.mjs';\n"} + report = self.tools(tree) + for path in ("tools/a.py", "tools/b.py", "tools/c.mjs", "tools/d.py"): + with self.subTest(tool=path): + self.assertTrue(report[path]["documented"]) + self.assertTrue(report[path]["referenced"]) + + def test_undocumented_and_unexercised_tools_fail(self): + result = run({"tools/lonely.py": "x = 1\n", "tools/README.md": "nothing here\n"}) + self.assertEqual(result["failing_tools"], ["tools/lonely.py"]) + self.assertFalse(result["tools"]["tools/lonely.py"]["documented"]) + self.assertFalse(result["tools"]["tools/lonely.py"]["referenced"]) + self.assertEqual(result["status"], "FAIL") + + def test_exercised_through_an_exercised_tool(self): + tree = {"tools/top.py": "import helper\n", "tools/helper.py": "x = 1\n", "tools/orphan.py": "x = 1\n", + "tools/README.md": "tools/top.py tools/helper.py tools/orphan.py\n", + "tests/test_top.py": "import top\n"} + report = self.tools(tree) + self.assertEqual(report["tools/helper.py"]["referenced_by"], ["via tools/top.py"]) + self.assertFalse(report["tools/orphan.py"]["referenced"]) + + def test_ambiguous_basename_needs_the_full_path(self): + tree = {"tools/run.py": "x = 1\n", "docs/USE.md": "call run.py\n", "tests/test_x.py": "import run\n"} + ambiguous = self.tools(tree, sorted(tree) + ["experiments/run.py"]) + self.assertFalse(ambiguous["tools/run.py"]["documented"]) + self.assertFalse(ambiguous["tools/run.py"]["referenced"]) + unique = self.tools(tree) + self.assertTrue(unique["tools/run.py"]["documented"]) + self.assertTrue(unique["tools/run.py"]["referenced"]) + + def test_full_path_counts_even_when_basename_is_ambiguous(self): + tree = {"tools/run.py": "x = 1\n", "docs/USE.md": "call tools/run.py\n", + ".github/workflows/ci.yml": " - run: python tools/run.py\n"} + report = self.tools(tree, sorted(tree) + ["experiments/run.py"]) + self.assertTrue(report["tools/run.py"]["documented"]) + self.assertTrue(report["tools/run.py"]["referenced"]) + + def test_module_docstring_is_recorded(self): + report = self.tools({"tools/a.py": '"""Does a."""\n', "tools/b.py": "x = 1\n"}) + self.assertEqual((report["tools/a.py"]["module_docstring"], report["tools/b.py"]["module_docstring"]), + (True, False)) + + def test_tests_and_init_files_are_not_tools(self): + report = self.tools({"tools/__init__.py": "", "tools/x/test_y.py": "x = 1\n", "tools/a.py": "x = 1\n"}) + self.assertEqual(sorted(report), ["tools/a.py"]) + + +class FailClosed(unittest.TestCase): + def test_undecodable_and_unparseable_production_files_fail(self): + for raw, expected in ((b"x = '\xff'\n", "undecodable"), (b"def broken(:\n", "unparseable")): + with self.subTest(raw=raw): + report = run({"tools/good.py": raw, **DOCUMENTED_TESTED}) + self.assertEqual(report["status"], "FAIL") + self.assertEqual([f["class"] for f in report["failing"]], [expected]) + + def test_unparseable_research_file_is_reported_not_failed(self): + report = run({"experiments/old.py": b"print 'python 2'\n"}) + self.assertEqual(classes(report), [("unparseable", 0)]) + self.assertEqual(report["status"], "PASS") + + +class CommandLine(unittest.TestCase): + def repo(self, files): + d = tempfile.mkdtemp() + self.addCleanup(lambda: subprocess.run(["rm", "-rf", d])) + env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@invalid", + GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="t@invalid") + subprocess.run(["git", "init", "-q", d], check=True) + for path, text in files.items(): + Path(d, path).parent.mkdir(parents=True, exist_ok=True) + Path(d, path).write_bytes(text) + subprocess.run(["git", "-C", d, "add", "-A"], check=True) + subprocess.run(["git", "-C", d, "commit", "-q", "-m", "fixture"], check=True, env=env) + return d + + def cli(self, root, *args): + script = str(ROOT / "tools" / "cbz" / "g11_census.py") + return subprocess.run([sys.executable, "-B", "-S", script, "--root", root, *args], + capture_output=True, text=True) + + def test_exit_codes_and_exact_committed_bytes(self): + good = {"tools/good.py": b"x = 1\n", "tools/README.md": b"tools/good.py\n", "tests/test_good.py": b"import good\n"} + root = self.repo(good) + self.assertEqual(self.cli(root).returncode, 0) + Path(root, "tools/good.py").write_bytes(b"def a():\n pass\n") # uncommitted: not read + self.assertEqual(self.cli(root).returncode, 0) + bad = self.repo({**good, "tools/good.py": b"def a():\n pass\n"}) + result = self.cli(bad) + self.assertEqual(result.returncode, 1) + self.assertIn("tools/good.py:1: tooling stub (a: pass)", result.stdout) + unexercised = self.repo({ + "tools/lonely.py": b"x = 1\n", + "tools/README.md": b"tools/lonely.py\n", + }) + unexercised_result = self.cli(unexercised) + self.assertEqual(unexercised_result.returncode, 1) + self.assertIn("tools/lonely.py: tool unexercised", unexercised_result.stdout) + self.assertEqual(self.cli(root, "--rev", "no-such-revision").returncode, 2) + + + + +class AdversarialControls(unittest.TestCase): + def test_decoy_names_do_not_exempt_stubs(self): + text = ( + "from abc import ABC as RealABC, abstractmethod as am\nimport typing as t\n" + "def abstractmethod(fn): return fn\n" + "class Fake:\n @staticmethod\n def abstractmethod(fn): return fn\n" + "class Protocol: pass\nclass ABC: pass\nclass ValueError: pass\n" + "@abstractmethod\ndef decoy_decorator():\n pass\n" + "@Fake.abstractmethod\ndef decoy_qualified():\n pass\n" + "class NotReallyError:\n def fail(self):\n pass\n" + "class NotReallyProtocol:\n def fail(self):\n ...\n" + "class NotReallyABC:\n def fail(self):\n pass\n" + "class StreamRequestHandler: pass\n" + "class NotReallyRequestHandler(StreamRequestHandler):\n def log_message(self, *args):\n pass\n" + "class DecoyProtocol(Protocol):\n def proto(self):\n pass\n" + "class DecoyABC(ABC):\n def abc(self):\n pass\n" + "class ShadowedBuiltin(ValueError):\n def builtin(self):\n pass\n" + "class Genuine:\n @am\n def abstract(self):\n pass\n" + "class Shape(t.Protocol):\n def area(self):\n ...\n" + "class RealBase(RealABC):\n @abstractmethod\n def decoy_method(self):\n pass\n" + ) + report = run({"tools/good.py": text, **DOCUMENTED_TESTED}) + self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]), + ["abc", "abstract", "builtin", "decoy_decorator", "decoy_method", "decoy_qualified", + "fail", "fail", "fail", "log_message", "proto"]) + self.assertEqual(report["status"], "FAIL") + local_shadow = ( + "from abc import abstractmethod\n" + "from typing import overload\n" + "def factory():\n" + " abstractmethod = lambda fn: fn\n" + " @abstractmethod\n" + " def local_decoy():\n" + " pass\n" + "def overloads():\n" + " overload = lambda fn: fn\n" + " @overload\n" + " def local_overload(x: int): ...\n" + " def local_overload(x):\n" + " return x\n" + ) + local_report = run({"tools/good.py": local_shadow, **DOCUMENTED_TESTED}) + self.assertEqual([(f["class"], f["detail"]) for f in local_report["findings"] + if f["class"].startswith("stub")], + [("stub", "local_decoy: pass"), ("stub", "local_overload: ellipsis")]) + + def test_comment_and_string_mentions_do_not_exercise_tools(self): + tree = { + "tools/runner.py": "def main():\n return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + "tests/test_runner.py": "# tools/runner.py\nDOC = 'tools/runner.py'\nprint('tools/runner.py')\n", + } + report = run(tree) + self.assertFalse(report["tools"]["tools/runner.py"]["referenced"]) + self.assertEqual(report["failing_tools"], ["tools/runner.py"]) + + python_cmd = run({ + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + "tests/test_runner.py": "import subprocess\nsubprocess.run(\"python -c 'tools/runner.py'\", shell=True)\n", + }) + self.assertFalse(python_cmd["tools"]["tools/runner.py"]["referenced"]) + + workflow_text = ("jobs:\n test:\n steps:\n - run: \"sh -c 'echo tools/runner.py'\"\n" + " - run: python3 -c tools/runner.py\n") + shell_cmd = run({ + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + ".github/workflows/ci.yml": workflow_text, + }) + self.assertEqual(g11._workflow_commands(workflow_text)[0][1], ["sh -c 'echo tools/runner.py'"]) + self.assertFalse(shell_cmd["tools"]["tools/runner.py"]["referenced"]) + + js_cmd = run({ + "tools/runner.mjs": "export const run = () => 1;\n", + "tools/README.md": "Use tools/runner.mjs.\n", + "tests/runner.test.mjs": "const sample = `\nimport '../tools/runner.mjs';\n`;\n", + }) + self.assertFalse(js_cmd["tools"]["tools/runner.mjs"]["referenced"]) + + def test_empty_and_comment_only_run_blocks_fail(self): + text = ("jobs:\n a:\n steps:\n" + " - run: |\n # only comment\n" + " - run: ''\n" + " - run: # empty\n" + " - run: true\n") + found = [f["line"] for f in run({".github/workflows/w.yml": text})["findings"] + if f["class"] == "noop"] + self.assertEqual(found, [4, 6, 7, 8]) + + def test_yaml_comments_are_scanned_for_markers(self): + text = ("name: ci\n# TODO workflow comment\njobs:\n a:\n steps:\n" + " - run: echo '# FIXME in data'\n") + report = run({".github/workflows/w.yml": text}) + self.assertEqual(classes(report), [("marker", 2), ("mention", 6)]) + + +class MutationAttribution(unittest.TestCase): + def test_mutation_kill_requires_exact_target_assertion(self): + expected = "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion" + good = {"parsed": True, "status": "FAILED", "exit": 1, "ran": 1, "failures": 1, + "errors": 0, "skipped": 0, "test_id": expected, + "traceback_method": "test_mutation_kill_requires_exact_target_assertion", + "traceback_line": 17, "exception_type": "AssertionError"} + self.assertTrue(g11m.qualifies_kill(good, expected, 17)) + wrong_line = dict(good, traceback_line=18) + self.assertFalse(g11m.qualifies_kill(wrong_line, expected, 17)) + wrong_test = dict(good, test_id=expected + ".other") + self.assertFalse(g11m.qualifies_kill(wrong_test, expected, 17)) + not_failed = dict(good, status="OK") + self.assertFalse(g11m.qualifies_kill(not_failed, expected, 17)) + has_errors = dict(good, errors=1) + self.assertFalse(g11m.qualifies_kill(has_errors, expected, 17)) + has_skips = dict(good, skipped=1) + self.assertFalse(g11m.qualifies_kill(has_skips, expected, 17)) + + expected_parser = "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion" + output = ( + "Ran 1 test in 0.001s\n\n" + "FAIL: test_mutation_kill_requires_exact_target_assertion (tests.test_g11_census.MutationAttribution)\n" + "Traceback (most recent call last):\n" + ' File "/tmp/tests/test_g11_census.py", line 123, in test_mutation_kill_requires_exact_target_assertion\n' + "AssertionError: expected\n\n" + "FAILED (failures=1)\n" + ) + proc = subprocess.CompletedProcess(["python"], 1, output, "") + parsed = g11m._parse_run(proc) + self.assertEqual(parsed["test_id"], expected_parser) + self.assertTrue(g11m.qualifies_kill(parsed, expected_parser, 123)) + current = output.replace("(tests.test_g11_census.MutationAttribution)", "(%s)" % expected_parser) + parsed_current = g11m._parse_run(subprocess.CompletedProcess(["python"], 1, current, "")) + self.assertEqual(parsed_current["test_id"], expected_parser) + subtest = current.replace("(%s)\n" % expected_parser, "(%s) (tool='tools/d.py')\n" % expected_parser) + parsed_subtest = g11m._parse_run(subprocess.CompletedProcess(["python"], 1, subtest, "")) + self.assertEqual(parsed_subtest["test_id"], expected_parser) + + + + + +class ReviewResidualControls(unittest.TestCase): + def test_stub_exemptions_are_immediate_and_scope_safe(self): + text = ( + "from abc import ABC, abstractmethod\n" + "from typing import Protocol, overload\n" + "from http.server import BaseHTTPRequestHandler\n" + "class Abstract(ABC):\n" + " @abstractmethod\n" + " def required(self): ...\n" + " def concrete(self): ...\n" + " class Nested:\n" + " def inner_abc(self): ...\n" + "class Interface(Protocol):\n" + " def declared(self): ...\n" + " class Nested:\n" + " def inner_protocol(self): ...\n" + "class Failure(Exception):\n" + " def exception_method(self): ...\n" + "def build(ABC, Exception, BaseHTTPRequestHandler):\n" + " class ShadowedABC(ABC):\n" + " @abstractmethod\n" + " def shadowed_abc(self): ...\n" + " class ShadowedError(Exception):\n" + " def shadowed_exception(self): ...\n" + " class ShadowedHandler(BaseHTTPRequestHandler):\n" + " def shadowed_handler_log_message(self, *args): ...\n" + "class Unpaired:\n" + " @abstractmethod\n" + " def abstract_without_abc(self): ...\n" + " @overload\n" + " def overload_without_implementation(self, value: int): ...\n" + ) + report = run({"tools/good.py": text, **DOCUMENTED_TESTED}) + stub_names = {f["detail"].split(":")[0] for f in report["findings"] if f["class"] == "stub"} + context_names = {f["detail"].split(":")[0] for f in report["findings"] if f["class"] == "stub-context"} + self.assertIn("required", context_names) + self.assertIn("declared", context_names) + self.assertIn("inner_abc", stub_names) + self.assertIn("shadowed_abc", stub_names) + for name in ("concrete", "inner_abc", "inner_protocol", "exception_method", + "shadowed_abc", "shadowed_exception", "shadowed_handler_log_message", + "abstract_without_abc", "overload_without_implementation"): + self.assertIn(name, stub_names) + self.assertNotIn(name, context_names) + + def test_tool_reachability_and_heredoc_boundaries(self): + uncalled = run({ + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + "tests/test_runner.py": "def unused_helper():\n import runner\n", + }) + self.assertFalse(uncalled["tools"]["tools/runner.py"]["referenced"]) + dead = run({ + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + "tests/test_runner.py": "if False:\n import runner\n", + }) + self.assertFalse(dead["tools"]["tools/runner.py"]["referenced"]) + reachable = run({ + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + "tests/test_runner.py": ( + "import unittest\n" + "class RunnerTests(unittest.TestCase):\n" + " def test_reaches_helper(self): self.helper()\n" + " def helper(self):\n" + " import runner\n" + ), + }) + self.assertTrue(reachable["tools"]["tools/runner.py"]["referenced"]) + decoy = run({ + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + "tests/test_runner.py": ( + "class TestCase:\n pass\n" + "class Fake(TestCase):\n" + " def test_decoy(self):\n" + " import runner\n" + ), + }) + self.assertFalse(decoy["tools"]["tools/runner.py"]["referenced"]) + imported_alias = run({ + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + "tests/test_runner.py": ( + "from unittest import TestCase as BaseCase\n" + "class RunnerTests(BaseCase):\n" + " def test_reaches_helper(self):\n" + " import runner\n" + ), + }) + self.assertTrue(imported_alias["tools"]["tools/runner.py"]["referenced"]) + workflow = ("jobs:\n test:\n steps:\n - run: |\n" + " cat <<'EOF'\n" + " python tools/runner.py\n" + " EOF\n") + heredoc = run({ + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + ".github/workflows/ci.yml": workflow, + }) + self.assertFalse(heredoc["tools"]["tools/runner.py"]["referenced"]) + real_command = run({ + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + ".github/workflows/ci.yml": workflow + " python tools/runner.py\n", + }) + self.assertTrue(real_command["tools"]["tools/runner.py"]["referenced"]) + + def test_mutation_inventory_is_sealed(self): + source = Path(g11m.__file__).read_bytes() + start = source.index(b"MUTATIONS = [") + end = source.index(b"\n\n\ndef _build_cases", start) + self.assertEqual(hashlib.sha256(source[start:end]).hexdigest(), + EXPECTED_MUTATION_INVENTORY_SHA256) + cases = g11m._build_cases() + self.assertEqual(len(cases), 75) + g11m._validate_mutation_inventory(cases) + altered_cases = (cases[:-1], cases + [cases[0]], cases[1:] + cases[:1]) + for altered in altered_cases: + with self.assertRaisesRegex(g11m.HarnessError, "mutation inventory"): + g11m._validate_mutation_inventory(altered) + weakened = list(cases) + original = weakened[0] + weakened[0] = (original[0], original[1], original[2], + original[3] + " # same-name weakening", original[4], original[5]) + with self.assertRaisesRegex(g11m.HarnessError, "mutation inventory mismatch"): + g11m._validate_mutation_inventory(tuple(weakened)) + + def test_mutation_child_environment_is_sanitized(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + tests_dir = root / "tests" + tests_dir.mkdir() + (tests_dir / "__init__.py").write_text("", encoding="utf-8") + (tests_dir / "test_probe.py").write_text( + "import os, unittest\n" + "class Probe(unittest.TestCase):\n" + " def test_environment(self):\n" + " for key in ('PYTHONPATH', 'PYTHONHOME', 'PYTHONPYCACHEPREFIX'):\n" + " self.assertNotIn(key, os.environ)\n" + " self.assertNotIn('poison', unittest.__file__)\n", + encoding="utf-8") + poison = root / "poison" + poison.mkdir() + (poison / "unittest.py").write_text("raise RuntimeError('poisoned unittest')\n", encoding="utf-8") + env = os.environ.copy() + env["PYTHONPATH"] = str(poison) + env["PYTHONHOME"] = str(poison) + env["PYTHONPYCACHEPREFIX"] = str(root / "pycache") + outcome = g11m._run(str(root), "tests.test_probe", env=env) + self.assertEqual(outcome.get("status"), "OK") + self.assertEqual(outcome.get("ran"), 1) + + def test_in_scope_tracked_symlink_is_rejected(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + subprocess.run(["git", "init", "-q", str(root)], check=True, capture_output=True) + subprocess.run(["git", "-C", str(root), "config", "user.name", "G11 Test"], + check=True, capture_output=True) + subprocess.run(["git", "-C", str(root), "config", "user.email", "g11@example.invalid"], + check=True, capture_output=True) + target = root / "target.txt" + target.write_text("not a scanned source\n", encoding="utf-8") + link = root / "tools" / "cbz" / "linked.py" + link.parent.mkdir(parents=True) + link.symlink_to(target) + subprocess.run(["git", "-C", str(root), "add", "tools/cbz/linked.py"], + check=True, capture_output=True) + subprocess.run(["git", "-C", str(root), "commit", "-qm", "symlink fixture"], + check=True, capture_output=True) + with self.assertRaisesRegex(ValueError, "in-scope non-regular tracked path"): + g11.read_tree(str(root), "HEAD") + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tools/cbz/g11_census.py b/tools/cbz/g11_census.py new file mode 100644 index 000000000..d2952d559 --- /dev/null +++ b/tools/cbz/g11_census.py @@ -0,0 +1,1011 @@ +#!/usr/bin/env python3 +"""CBZ-0 gate G11 census: stub scan and tool documentation check (stdlib only). + +Reads the exact blob bytes of one git revision and reports: + + stub a Python function whose body, after any docstring, is only `pass`, + `...` or `raise NotImplementedError`, or is the docstring alone. + @abstractmethod is contextual only in an immediate imported ABC subclass; + @overload requires a complete overload group and a concrete implementation; + immediate Protocol methods are contextual; ABC or exception inheritance alone + is not an exemption. + marker TODO, FIXME, XXX, TBD or STUB (upper case) or "not implemented" in + a code comment. In Python the comment is found by tokenizing; in + .mjs/.js/.cjs/.sh a comment is found by a quote-parity heuristic. + The same words in string data are recorded as mentions only. + noop a workflow step whose `run:` does nothing: `true`, `:` or `exit 0`. + tools every executable source under tools/ (not tests, not __init__.py), + with tracked Markdown documentation and statically reachable test + or workflow references. References are not proof of runtime execution. + +Every path has an area. Only `tooling` (tools/, .github/) and `site` +(docs/site/ outside vendor/) are production; findings elsewhere are reported +and never fail. A production file that is not UTF-8, or Python that does not +parse, fails closed. + +Exit status: 0 when nothing in production fails, 1 when something does, 2 when +the revision cannot be read. The census is evidence for a reviewer, never a +G11 certificate. +""" +from __future__ import annotations + +import argparse +import ast +import builtins +import posixpath +import shlex +import io +import json +import re +import subprocess +import sys +import tokenize + +MARKER = re.compile(r"\b(?:TODO|FIXME|XXX|TBD|STUB)\b|\b(?i:not implemented)\b") +CODE_SUFFIXES = (".py", ".mjs", ".js", ".cjs", ".sh") +SCANNED = CODE_SUFFIXES + (".md", ".yml", ".yaml") +LEGIT_DECORATORS = {"abstractmethod", "overload"} +LEGIT_BASES = {"Protocol", "ABC"} +NOOP_RUN = {"true", ":", "exit 0"} +PRODUCTION = {"tooling", "site"} + + +def area(path): + name = path.rsplit("/", 1)[-1] + if (path.startswith("tests/") or "/tests/" in path or name == "conftest.py" + or name.startswith("test_") or name.endswith("_test.py") or ".test." in name): + return "tests" + if path.startswith("tools/") or path.startswith(".github/"): + return "tooling" + if path.startswith("docs/site/vendor/"): + return "vendor" + if path.startswith("docs/site/"): + return "site" + if path.startswith("experiments/"): + return "research" + return "evidence" + + +def _decorator_name(node): + while isinstance(node, ast.Call): + node = node.func + if isinstance(node, ast.Attribute): + return node.attr + if isinstance(node, ast.Name): + return node.id + return "" + + +def _base_name(node): + if isinstance(node, ast.Subscript): + node = node.value + if isinstance(node, ast.Attribute): + return node.attr + if isinstance(node, ast.Name): + return node.id + return "" + +def _dotted_name(node): + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + base = _dotted_name(node.value) + return base + "." + node.attr if base else "" + return "" + + +def _target_names(node): + if isinstance(node, ast.Name): + return {node.id} + if isinstance(node, (ast.Tuple, ast.List)): + names = set() + for item in node.elts: + names.update(_target_names(item)) + return names + if isinstance(node, ast.Starred): + return _target_names(node.value) + return set() + + +def _scope_bindings(scope): + """Names bound in one lexical scope; nested scope bodies are kept separate.""" + names = set() + if isinstance(scope, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + args = scope.args + for arg in args.posonlyargs + args.args + args.kwonlyargs: + names.add(arg.arg) + if args.vararg: + names.add(args.vararg.arg) + if args.kwarg: + names.add(args.kwarg.arg) + + def visit(node, root=False): + if not root and isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(node.name) + return + if not root and isinstance(node, ast.Lambda): + return + if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)): + names.add(node.id) + elif isinstance(node, ast.Import): + for alias in node.names: + names.add(alias.asname or alias.name.split(".", 1)[0]) + elif isinstance(node, ast.ImportFrom): + for alias in node.names: + if alias.name != "*": + names.add(alias.asname or alias.name) + elif isinstance(node, ast.ExceptHandler) and node.name: + names.add(node.name) + for child in ast.iter_child_nodes(node): + visit(child) + + visit(scope, True) + return names + + +def _import_bindings(tree): + modules, symbols, shadowed = {}, {}, set() + for node in tree.body: + if type(node) is ast.Import: + for alias in node.names: + bound = alias.asname or alias.name.split(".", 1)[0] + modules[bound] = alias.name if alias.asname else alias.name.split(".", 1)[0] + elif type(node) is ast.ImportFrom and node.level == 0: + for alias in node.names: + symbols[alias.asname or alias.name] = (node.module or "", alias.name) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + shadowed.add(node.name) + elif isinstance(node, ast.Assign): + for target in node.targets: + shadowed.update(_target_names(target)) + elif isinstance(node, (ast.AnnAssign, ast.AugAssign)): + shadowed.update(_target_names(node.target)) + for name in shadowed: + modules.pop(name, None) + symbols.pop(name, None) + return {"modules": modules, "symbols": symbols, "shadowed": shadowed} + + +def _resolve_import(node, imports): + if isinstance(node, ast.Name): + if node.id in imports["shadowed"]: + return None + return imports["symbols"].get(node.id) + dotted = _dotted_name(node) + if not dotted: + return None + parts = dotted.split(".") + if parts[0] in imports["shadowed"]: + return None + module = imports["modules"].get(parts[0]) + if module is None or len(parts) < 2: + return None + resolved = module + "." + ".".join(parts[1:]) + return tuple(resolved.rsplit(".", 1)) + + +def _resolve_scoped_import(node, imports, scope_shadowed=()): + target = node + while isinstance(target, (ast.Call, ast.Subscript)): + target = target.func if isinstance(target, ast.Call) else target.value + dotted = _dotted_name(target) + binding = dotted.split(".", 1)[0] if dotted else "" + if binding and binding in scope_shadowed: + return None + return _resolve_import(target, imports) + + +def _trusted_decorator(node, imports, scope_shadowed=()): + resolved = _resolve_scoped_import(node, imports, scope_shadowed) + return resolved in {("abc", "abstractmethod"), ("typing", "overload"), + ("typing_extensions", "overload")} + + +def _overload_declaration(node, imports, scope_shadowed=()): + return ( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and any(_resolve_scoped_import(dec, imports, scope_shadowed) in { + ("typing", "overload"), ("typing_extensions", "overload") + } for dec in node.decorator_list) + ) + + +def _complete_overload_group(func, siblings, imports, scope_shadowed=()): + if not _overload_declaration(func, imports, scope_shadowed): + return False + try: + index = next(i for i, node in enumerate(siblings) if node is func) + except StopIteration: + return False + start = index + while (start > 0 and isinstance(siblings[start - 1], (ast.FunctionDef, ast.AsyncFunctionDef)) + and siblings[start - 1].name == func.name + and _overload_declaration(siblings[start - 1], imports, scope_shadowed)): + start -= 1 + end = start + while (end < len(siblings) and isinstance(siblings[end], (ast.FunctionDef, ast.AsyncFunctionDef)) + and siblings[end].name == func.name + and _overload_declaration(siblings[end], imports, scope_shadowed)): + end += 1 + if not any(siblings[i] is func for i in range(start, end)) or end >= len(siblings): + return False + implementation = siblings[end] + return ( + isinstance(implementation, (ast.FunctionDef, ast.AsyncFunctionDef)) + and implementation.name == func.name + and not _overload_declaration(implementation, imports, scope_shadowed) + and _stub_kind(implementation.body) is None + ) + + +def _trusted_abstract_base(node, imports, scope_shadowed=()): + resolved = _resolve_scoped_import(node, imports, scope_shadowed) + if resolved == ("abc", "ABC"): + return "ABC" + if resolved in {("typing", "Protocol"), ("typing_extensions", "Protocol")}: + return "Protocol" + return None + + +def _builtin_exception_base(node, imports, scope_shadowed=()): + resolved = _resolve_scoped_import(node, imports, scope_shadowed) + name = None + if resolved and resolved[0] == "builtins": + name = resolved[1] + elif (isinstance(node, ast.Name) and node.id in builtins.__dict__ + and node.id not in imports["shadowed"] and node.id not in scope_shadowed): + name = node.id + if not name: + return False + candidate = getattr(builtins, name, None) + return isinstance(candidate, type) and issubclass(candidate, BaseException) + + +def _http_request_handler_base(node, imports, scope_shadowed=()): + return _resolve_scoped_import(node, imports, scope_shadowed) in { + ("http.server", "BaseHTTPRequestHandler"), + ("http.server", "SimpleHTTPRequestHandler"), + ("http.server", "CGIHTTPRequestHandler"), + } + + + + +def _stub_kind(body): + rest = body + if rest and isinstance(rest[0], ast.Expr) and isinstance(rest[0].value, ast.Constant) \ + and isinstance(rest[0].value.value, str): + rest = rest[1:] + if not rest: + return "docstring only" + if len(rest) != 1: + return None + stmt = rest[0] + if isinstance(stmt, ast.Pass): + return "pass" + if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Constant) and stmt.value.value is Ellipsis: + return "ellipsis" + if isinstance(stmt, ast.Raise) and stmt.exc is not None: + exc = stmt.exc.func if isinstance(stmt.exc, ast.Call) else stmt.exc + if isinstance(exc, ast.Name) and exc.id == "NotImplementedError": + return "raise NotImplementedError" + return None + + +def _legit_context(func, classes, imports, scope_shadowed=(), siblings=()): + for dec in func.decorator_list: + if _trusted_decorator(dec, imports, scope_shadowed): + resolved = _resolve_scoped_import(dec, imports, scope_shadowed) + if resolved == ("abc", "abstractmethod"): + if classes and any(_trusted_abstract_base(base, imports, scope_shadowed) == "ABC" + for base in classes[-1].bases): + return "@abstractmethod" + elif (resolved in {("typing", "overload"), ("typing_extensions", "overload")} + and _complete_overload_group(func, siblings, imports, scope_shadowed)): + return "@overload" + # Only the immediate class can supply method context. + for cls in classes[-1:]: + for base in cls.bases: + base_kind = _trusted_abstract_base(base, imports, scope_shadowed) + if base_kind == "Protocol": + return "method of a Protocol class" + if base_kind == "ABC": + continue + if _builtin_exception_base(base, imports, scope_shadowed): + continue + if (_http_request_handler_base(base, imports, scope_shadowed) + and func.name == "log_message"): + return "log_message override that silences http.server request logging" + return None + + +def python_stubs(tree, imports=None): + found = [] + if imports is None: + imports = _import_bindings(tree) + + def visit(node, classes, scope_shadowed, siblings): + for child in ast.iter_child_nodes(node): + if isinstance(child, ast.ClassDef): + visit(child, classes + [child], scope_shadowed | _scope_bindings(child), child.body) + elif isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): + kind = _stub_kind(child.body) + if kind: + found.append({"line": child.lineno, "name": child.name, "kind": kind, + "context": _legit_context(child, classes, imports, scope_shadowed, siblings)}) + visit(child, [], scope_shadowed | _scope_bindings(child), child.body) + else: + child_body = getattr(child, "body", None) + visit(child, classes, scope_shadowed, + child_body if isinstance(child_body, list) else siblings) + visit(tree, [], set(), tree.body) + return found + + +def python_words(raw): + markers, mentions = [], [] + for tok in tokenize.tokenize(io.BytesIO(raw).readline): + if tok.type == tokenize.COMMENT: + target = markers + elif tok.type == tokenize.STRING or tok.type == getattr(tokenize, "FSTRING_MIDDLE", -1): + target = mentions + else: + continue + for m in MARKER.finditer(tok.string): + target.append({"line": tok.start[0] + tok.string[:m.start()].count("\n"), "word": m.group(0)}) + return markers, mentions + + +def _comment_start(line, opener): + # Index where a comment opens outside quotes, by quote parity; -1 if none. + quote = None + i = 0 + while i < len(line): + ch = line[i] + if quote: + if ch == "\\": + i += 2 + continue + if ch == quote: + quote = None + elif ch in "\"'`": + quote = ch + elif line.startswith(opener, i) and (opener != "#" or i == 0 or line[i - 1].isspace()): + return i + i += 1 + return -1 + + +def text_words(text, path): + markers, mentions = [], [] + opener = "#" if path.endswith(".sh") else "//" + in_block = False + for number, line in enumerate(text.splitlines(), 1): + comment_parts, data_parts = [], [] + if opener == "//": + rest = line + while rest: + if in_block: + end = rest.find("*/") + comment_parts.append(rest if end < 0 else rest[:end]) + if end < 0: + rest = "" + else: + in_block, rest = False, rest[end + 2:] + continue + line_at, block_at = _comment_start(rest, "//"), _comment_start(rest, "/*") + starts = [p for p in (line_at, block_at) if p >= 0] + if not starts: + data_parts.append(rest) + break + first = min(starts) + data_parts.append(rest[:first]) + if first == line_at: + comment_parts.append(rest[first:]) + break + in_block, rest = True, rest[first + 2:] + else: + at = _comment_start(line, "#") + data_parts.append(line if at < 0 else line[:at]) + if at >= 0: + comment_parts.append(line[at:]) + for part in comment_parts: + markers.extend({"line": number, "word": m.group(0)} for m in MARKER.finditer(part)) + for part in data_parts: + mentions.extend({"line": number, "word": m.group(0)} for m in MARKER.finditer(part)) + return markers, mentions + + + +def _yaml_comment_start(line): + quote = None + i = 0 + while i < len(line): + ch = line[i] + if quote: + if quote == '"' and ch == "\\": + i += 2 + continue + if quote == "'" and ch == "'" and i + 1 < len(line) and line[i + 1] == "'": + i += 2 + continue + if ch == quote: + quote = None + elif ch in "\"'": + quote = ch + elif ch == "#" and (i == 0 or line[i - 1].isspace()): + return i + i += 1 + return -1 + + +def yaml_words(text): + markers, mentions = [], [] + for number, line in enumerate(text.splitlines(), 1): + at = _yaml_comment_start(line) + data = line if at < 0 else line[:at] + comment = "" if at < 0 else line[at + 1:] + mentions.extend({"line": number, "word": m.group(0)} for m in MARKER.finditer(data)) + markers.extend({"line": number, "word": m.group(0)} for m in MARKER.finditer(comment)) + return markers, mentions + + +def _strip_shell_comment(line): + at = _comment_start(line, "#") + return line if at < 0 else line[:at] + + +def _workflow_commands(text): + found, lines = [], text.splitlines() + for number, line in enumerate(lines, 1): + m = re.match(r"^(\s*)(?:-\s+)?run:\s*(.*?)\s*$", line) + if not m: + continue + value = m.group(2).strip() + if value.startswith("#"): + value = "" + if not value or value[0] in "|>": + key_column = line.index("run:") + body = [] + for nxt in lines[number:]: + if nxt.strip() and len(nxt) - len(nxt.lstrip()) <= key_column: + break + body.append(nxt.strip()) + if value and value[0] in "|>": + commands = [b for b in body if b and not b.startswith("#")] + commands = [_strip_shell_comment(b).strip() for b in commands] + commands = [b for b in commands if b] + else: + commands = [] + else: + if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"": + value = value[1:-1] + commands = [_strip_shell_comment(value).strip()] + commands = [_strip_shell_comment(c).strip() for c in commands] + commands = [c for c in commands if c] + found.append((number, commands)) + return found + + +def noop_steps(text): + found = [] + for number, commands in _workflow_commands(text): + if not commands: + found.append({"line": number, "run": ""}) + if commands and all(c in NOOP_RUN for c in commands): + found.append({"line": number, "run": " ; ".join(commands)}) + return found + + +def tool_paths(files): + return sorted(p for p in files if p.startswith("tools/") and p.endswith(CODE_SUFFIXES) + and area(p) == "tooling" and not p.endswith("__init__.py")) + + +def _module(path): + return path.rsplit("/", 1)[-1].rsplit(".", 1)[0] + + +def _names_tool(text, path, unique): + # The full path always counts. The module name and the basename count only + # when no other tracked file shares the basename, so neither is ambiguous. + pattern = r"\b" + re.escape(path) + r"\b" + if unique: + module = re.escape(_module(path)) + pattern += (r"|\b(?:import|from)\s+(?:[\w.]+\.)?" + module + r"\b" + r"|\bfrom\s+[\w.]+\s+import\s+(?:\([^)]*|[^\n]*)\b" + module + r"\b" + r"|\b" + re.escape(path.rsplit("/", 1)[-1]) + r"\b") + return re.search(pattern, text) is not None + + + +def _tool_for_module(module, tools, unique): + if not module: + return None + module = module.strip(".") + candidate = module.replace(".", "/") + ".py" + if candidate in tools: + return candidate + basename = module.rsplit(".", 1)[-1] + if unique.get(basename): + for path in tools: + if path.rsplit("/", 1)[-1] == basename + ".py": + return path + return None + + +def _tool_for_token(token, tools): + token = token.strip().strip("'\"").replace("\\", "/") + while token.startswith("./"): + token = token[2:] + if token in tools: + return token + for path in tools: + if token.endswith("/" + path): + return path + return None + + +def _literal_bool(node): + try: + return bool(ast.literal_eval(node)) + except (ValueError, TypeError): + return None + + +def _is_main_guard(node): + return (isinstance(node, ast.Compare) and isinstance(node.left, ast.Name) + and node.left.id == "__name__" and len(node.ops) == 1 + and isinstance(node.ops[0], ast.Eq) and len(node.comparators) == 1 + and isinstance(node.comparators[0], ast.Constant) + and node.comparators[0].value == "__main__") + + +def _reachable_python_nodes(tree, mode): + functions, methods = {}, {} + + def register(body, owner=None): + for statement in body: + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + if owner is None: + functions.setdefault(statement.name, statement) + else: + methods.setdefault((owner, statement.name), statement) + elif isinstance(statement, ast.ClassDef): + register(statement.body, statement.name) + + register(tree.body) + test_classes = set() + if mode == "tests": + imports = _import_bindings(tree) + for statement in tree.body: + if (isinstance(statement, ast.ClassDef) + and any(_resolve_import(base, imports) == ("unittest", "TestCase") + for base in statement.bases)): + test_classes.add(statement.name) + + nodes, queue, visited = [], [], set() + + def visit(node, owner=None): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + # A definition is not a call: only decorators/defaults execute here. + for decorator in node.decorator_list: + visit(decorator, owner) + for default in node.args.defaults: + visit(default, owner) + for default in node.args.kw_defaults: + if default is not None: + visit(default, owner) + if mode == "tests" and owner in test_classes and node.name.startswith("test_"): + queue.append((node, owner)) + return + if isinstance(node, ast.ClassDef): + for decorator in node.decorator_list: + visit(decorator, owner) + for base in node.bases: + visit(base, owner) + for keyword in node.keywords: + visit(keyword.value, owner) + for statement in node.body: + visit(statement, node.name) + return + if isinstance(node, ast.If): + visit(node.test, owner) + if _is_main_guard(node.test): + branch = node.body if mode == "script" else node.orelse + else: + truth = _literal_bool(node.test) + if truth is None: + return + branch = node.body if truth else node.orelse + for statement in branch: + visit(statement, owner) + return + if isinstance(node, (ast.For, ast.AsyncFor, ast.While)): + if isinstance(node, (ast.For, ast.AsyncFor)): + visit(node.iter, owner) + else: + visit(node.test, owner) + return + + nodes.append(node) + if isinstance(node, ast.Call): + target = node.func + if isinstance(target, ast.Name) and target.id in functions: + queue.append((functions[target.id], None)) + elif (isinstance(target, ast.Attribute) and isinstance(target.value, ast.Name) + and target.value.id in {"self", "cls"} and owner): + method = methods.get((owner, target.attr)) + if method: + queue.append((method, owner)) + for child in ast.iter_child_nodes(node): + visit(child, owner) + + for statement in tree.body: + visit(statement) + while queue: + function, owner = queue.pop() + identity = (id(function), owner) + if identity in visited: + continue + visited.add(identity) + for statement in function.body: + visit(statement, owner) + return nodes + + +def python_tool_references(text, tools, unique, mode="script"): + references = set() + try: + tree = ast.parse(text) + except (SyntaxError, ValueError): + return references + for node in _reachable_python_nodes(tree, mode): + if isinstance(node, ast.Import): + for alias in node.names: + path = _tool_for_module(alias.name, tools, unique) + if path: + references.add(path) + elif isinstance(node, ast.ImportFrom) and node.level == 0: + module = node.module or "" + path = _tool_for_module(module, tools, unique) + if path: + references.add(path) + else: + for alias in node.names: + path = _tool_for_module((module + "." + alias.name).strip("."), tools, unique) + if path: + references.add(path) + elif isinstance(node, ast.Call): + call_name = _dotted_name(node.func) + if call_name in {"subprocess.run", "subprocess.call", "subprocess.Popen", + "subprocess.check_call", "subprocess.check_output", "os.system", + "runpy.run_path"}: + for arg in node.args: + if isinstance(arg, (ast.List, ast.Tuple)): + tokens = [elt.value for elt in arg.elts + if isinstance(elt, ast.Constant) and isinstance(elt.value, str)] + elif isinstance(arg, ast.Constant) and isinstance(arg.value, str): + try: + tokens = shlex.split(arg.value) + except ValueError: + tokens = [] + else: + tokens = [] + if tokens and tokens[0] in {"python", "python3", "python3.11", "bash", "sh"}: + if "-c" in tokens[1:]: + continue + for token in tokens[1:]: + if token == "-m": + continue + path = _tool_for_token(token, tools) + if path: + references.add(path) + break + elif tokens: + path = _tool_for_token(tokens[0], tools) + if path: + references.add(path) + return references + + +def _strip_js_comments(text): + out = [] + i = 0 + quote = None + line_comment = False + block_comment = False + while i < len(text): + ch = text[i] + nxt = text[i + 1] if i + 1 < len(text) else "" + if line_comment: + if ch == "\n": + line_comment = False + out.append(ch) + else: + out.append(" ") + i += 1 + continue + if block_comment: + if ch == "*" and nxt == "/": + out.extend((" ", " ")) + block_comment = False + i += 2 + else: + out.append("\n" if ch == "\n" else " ") + i += 1 + continue + if quote: + if quote == "`": + if ch == "\\" and i + 1 < len(text): + out.extend((" ", " ")) + i += 2 + continue + if ch == quote: + quote = None + out.append("\n" if ch == "\n" else " ") + i += 1 + continue + out.append(ch) + if ch == "\\" and i + 1 < len(text): + out.append(text[i + 1]) + i += 2 + continue + if ch == quote: + quote = None + i += 1 + continue + if ch in "\"'" + chr(96): + quote = ch + out.append(" " if ch == chr(96) else ch) + i += 1 + continue + if ch == "/" and nxt == "/": + line_comment = True + out.extend((" ", " ")) + i += 2 + continue + if ch == "/" and nxt == "*": + block_comment = True + out.extend((" ", " ")) + i += 2 + continue + out.append(ch) + i += 1 + return "".join(out) + + +def js_tool_references(text, source_path, tools): + references = set() + visible = _strip_js_comments(text) + pattern = re.compile(r"(?m)^\s*import\s+(?:[^'\";]+?\s+from\s+)?['\"]([^'\"]+)['\"]\s*;?") + for match in pattern.finditer(visible): + specifier = match.group(1) + if specifier.startswith("."): + target = posixpath.normpath(posixpath.join(posixpath.dirname(source_path), specifier)) + else: + target = specifier + if target in tools: + references.add(target) + return references + + +def _heredoc_opener(command): + match = re.search(r"<<(?P-?)(?:'(?P[^']+)'|\"(?P[^\"]+)\"|(?P[A-Za-z_][A-Za-z0-9_]*))", + command) + if not match: + return None + delimiter = match.group("single") or match.group("double") or match.group("bare") + return delimiter, match.group("strip") == "-" + + +def workflow_tool_references(text, tools): + references = set() + for _number, commands in _workflow_commands(text): + heredoc = None + for command in commands: + if heredoc: + delimiter, strip_tabs = heredoc + candidate = command.lstrip("\t") if strip_tabs else command + if candidate.strip() == delimiter: + heredoc = None + continue + try: + tokens = shlex.split(command) + except ValueError: + tokens = [] + if tokens: + if tokens[0] in {"python", "python3", "python3.11", "bash", "sh"}: + if "-c" not in tokens[1:]: + for index, token in enumerate(tokens[1:], 1): + if token == "-m" and index + 1 < len(tokens): + module_path = tokens[index + 1].replace(".", "/") + ".py" + if module_path in tools: + references.add(module_path) + break + if token.startswith("-"): + continue + path = _tool_for_token(token, tools) + if path: + references.add(path) + break + else: + path = _tool_for_token(tokens[0], tools) + if path: + references.add(path) + opener = _heredoc_opener(command) + if opener: + heredoc = opener + return references + + +def _markdown_visible(text): + return re.sub(r"", "", text, flags=re.S) + + +def tool_report(files, texts, all_paths): + tools = tool_paths(files) + basenames = {} + for p in all_paths: + name = p.rsplit("/", 1)[-1] + basenames[name] = basenames.get(name, 0) + 1 + unique = {p: basenames[p.rsplit("/", 1)[-1]] == 1 for p in tools} + unique_modules = {p.rsplit("/", 1)[-1].rsplit(".", 1)[0]: unique[p] for p in tools} + docs = [p for p in texts if p.endswith(".md")] + exercisers = [p for p in texts if area(p) == "tests" or + (p.startswith(".github/workflows/") and p.endswith((".yml", ".yaml")))] + direct, tool_dependencies = {p: [] for p in tools}, {} + for path in tools: + tool_dependencies[path] = python_tool_references(texts.get(path, ""), tools, unique_modules, mode="script") + for exerciser in exercisers: + if exerciser.endswith(".py"): + refs = python_tool_references(texts[exerciser], tools, unique_modules, mode="tests") + elif exerciser.endswith((".mjs", ".js", ".cjs")): + refs = js_tool_references(texts[exerciser], exerciser, tools) + else: + refs = workflow_tool_references(texts[exerciser], tools) + for path in refs: + direct[path].append(exerciser) + report = {} + for path in tools: + documented = [d for d in docs if _names_tool(_markdown_visible(texts[d]), path, unique[path])] + direct_paths = direct[path] + docstring = None + if path.endswith(".py") and path in texts: + try: + docstring = bool(ast.get_docstring(ast.parse(files[path]))) + except (SyntaxError, ValueError): + docstring = False + report[path] = {"documented_by": documented[:5], "documented": bool(documented), + "referenced_by": direct_paths[:5], "referenced": bool(direct_paths), + "module_docstring": docstring} + changed = True + while changed: + changed = False + for path in tools: + if report[path]["referenced"] or path not in texts: + continue + for other in tools: + if other != path and report[other]["referenced"] and other in texts and path in tool_dependencies.get(other, set()): + report[path]["referenced"] = True + report[path]["referenced_by"] = ["via " + other] + changed = True + break + return report + + +def census(files, all_paths=None): + """Census of a {path: bytes} tree. Pure: no I/O. `all_paths` lists every + tracked path (default: the keys of `files`) for basename ambiguity.""" + findings, texts = [], {} + for path in sorted(files): + raw = files[path] + kind = area(path) + if not path.endswith(SCANNED): + continue + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + findings.append({"path": path, "area": kind, "class": "undecodable", "line": 0, "detail": "not UTF-8"}) + continue + texts[path] = text + if path.endswith(".py"): + try: + tree = ast.parse(raw, filename=path) + markers, mentions = python_words(raw) + except (SyntaxError, ValueError, tokenize.TokenError) as exc: + findings.append({"path": path, "area": kind, "class": "unparseable", "line": 0, "detail": str(exc)[:120]}) + continue + imports = _import_bindings(tree) + for stub in python_stubs(tree, imports): + findings.append({"path": path, "area": kind, "class": "stub" if not stub["context"] else "stub-context", + "line": stub["line"], "detail": "%s: %s%s" % (stub["name"], stub["kind"], + "; " + stub["context"] if stub["context"] else "")}) + elif path.endswith((".mjs", ".js", ".cjs", ".sh")): + markers, mentions = text_words(text, path) + elif path.endswith((".yml", ".yaml")): + markers, mentions = yaml_words(text) + else: + markers, mentions = [], [] + findings.extend({"path": path, "area": kind, "class": "marker", "line": m["line"], "detail": m["word"]} for m in markers) + findings.extend({"path": path, "area": kind, "class": "mention", "line": m["line"], "detail": m["word"]} for m in mentions) + if path.startswith(".github/workflows/") and path.endswith((".yml", ".yaml")): + findings.extend({"path": path, "area": kind, "class": "noop", "line": n["line"], "detail": n["run"]} + for n in noop_steps(text)) + tools = tool_report(files, texts, sorted(all_paths if all_paths is not None else files)) + failing = [f for f in findings if f["area"] in PRODUCTION + and f["class"] in {"stub", "marker", "noop", "undecodable", "unparseable"}] + failing_tools = sorted(p for p, t in tools.items() if not (t["documented"] and t["referenced"])) + return {"findings": findings, "tools": tools, "failing": failing, "failing_tools": failing_tools, + "status": "FAIL" if failing or failing_tools else "PASS"} + + +def read_tree(root, rev): + listing = subprocess.run(["git", "-C", root, "ls-tree", "-r", "-z", "--full-tree", rev], + check=True, capture_output=True).stdout + entries, skipped, tracked = [], [], [] + for item in listing.split(b"\0"): + if not item: + continue + meta, path = item.split(b"\t", 1) + mode, kind, sha = meta.split() + path = path.decode("utf-8", "surrogateescape") + if path.endswith(SCANNED) and (kind != b"blob" or mode == b"120000"): + raise ValueError("in-scope non-regular tracked path %s (mode %s, type %s)" % + (path, mode.decode(), kind.decode())) + if kind == b"blob" and mode != b"120000": + if path.endswith(SCANNED): + entries.append((path, sha.decode())) + tracked.append(path) + else: + skipped.append({"path": path, "mode": mode.decode(), "type": kind.decode()}) + batch = subprocess.run(["git", "-C", root, "cat-file", "--batch"], check=True, capture_output=True, + input=b"".join(sha.encode() + b"\n" for _p, sha in entries)).stdout + files, at = {}, 0 + for path, sha in entries: + end = batch.index(b"\n", at) + header = batch[at:end].split() + if len(header) != 3 or header[0].decode() != sha: + raise ValueError("unexpected git cat-file header for " + path) + size = int(header[2]) + files[path] = batch[end + 1:end + 1 + size] + at = end + 1 + size + 1 + return files, skipped, tracked + + +def main(argv=None): + parser = argparse.ArgumentParser(description="CBZ-0 G11 census: stub scan and tool documentation check.") + parser.add_argument("--root", default=".", help="git repository (default: current directory)") + parser.add_argument("--rev", default="HEAD", help="revision to read (default: HEAD)") + parser.add_argument("--json", help="write the full report to this file") + args = parser.parse_args(argv) + try: + commit = subprocess.run(["git", "-C", args.root, "rev-parse", "--verify", args.rev + "^{commit}"], + check=True, capture_output=True, text=True).stdout.strip() + files, skipped, tracked = read_tree(args.root, commit) + except (subprocess.CalledProcessError, OSError, ValueError) as exc: + print("G11_CENSUS_ERROR: cannot read revision %s: %s" % (args.rev, exc), file=sys.stderr) + return 2 + report = census(files, tracked) + report.update(commit=commit, files=len(files), tracked=len(tracked), skipped=skipped) + if args.json: + with open(args.json, "w", encoding="utf-8") as fh: + json.dump(report, fh, indent=1, sort_keys=True) + fh.write("\n") + for f in report["failing"]: + print("%s:%d: %s %s (%s)" % (f["path"], f["line"], f["area"], f["class"], f["detail"])) + for path in report["failing_tools"]: + t = report["tools"][path] + print("%s: tool %s" % (path, ", ".join(w for w, ok in (("undocumented", t["documented"]), + ("unexercised", t["referenced"])) if not ok))) + counts = {} + for f in report["findings"]: + counts[f["class"]] = counts.get(f["class"], 0) + 1 + print("G11_CENSUS_%s: commit %s, %d files, %d tools, findings %s, failing %d, failing tools %d" + % (report["status"], commit, len(files), len(report["tools"]), json.dumps(counts, sort_keys=True), + len(report["failing"]), len(report["failing_tools"]))) + return 1 if report["status"] == "FAIL" else 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py new file mode 100644 index 000000000..27688671c --- /dev/null +++ b/tools/cbz/g11_mutation_selftest.py @@ -0,0 +1,608 @@ +#!/usr/bin/env python3 +"""Mutation self-test of the G11 census (tools/cbz/g11_census.py). + +Each mutation weakens the census or its attribution guard in one way. For +every mutation the harness copies the census, test module and harness into an +isolated directory, applies exactly one mutation, and runs one designated test. +A mutation is KILLED only when the clean full baseline passes and that one test +reports exactly one AssertionError at its exact designated assertion line, +with the exact module/class/method identity, FAILED status, zero errors and +zero skips. Setup, import, teardown, cleanup, process and unrelated assertion +failures never count as kills. The self-test passes only when the unmutated +baseline passes and every mutation is killed. + +A mutation whose target text does not occur exactly once is a harness error, +never a silent no-op. + +Usage: python3 -B -S tools/cbz/g11_mutation_selftest.py [--root DIR] [--json] +Exit status: 0 pass, 1 a mutation survived or the baseline failed, 2 harness error. +""" +from __future__ import annotations + +import argparse +import ast +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +TOOL = os.path.join("tools", "cbz", "g11_census.py") +TESTS = os.path.join("tests", "test_g11_census.py") +HARNESS = os.path.join("tools", "cbz", "g11_mutation_selftest.py") + +MUTATIONS = [ + ("pass not detected", ' if isinstance(stmt, ast.Pass):\n return "pass"\n', + ' if False:\n return "pass"\n'), + ("ellipsis not detected", "and stmt.value.value is Ellipsis:", "and stmt.value.value is None:"), + ("NotImplementedError not detected", 'exc.id == "NotImplementedError"', 'exc.id == "NotImplemented"'), + ("raise call not unwrapped", "exc = stmt.exc.func if isinstance(stmt.exc, ast.Call) else stmt.exc", + "exc = stmt.exc"), + ("docstring-only not reported", ' if not rest:\n return "docstring only"\n', + " if not rest:\n return None\n"), + ("docstring not stripped", " rest = rest[1:]\n", " rest = rest[0:]\n"), + ("decorator context ignored", " if _trusted_decorator(dec, imports, scope_shadowed):", " if False:"), + ("Protocol/ABC context ignored", ' if base_kind == "Protocol":', " if False:"), + ("exception context ignored", + " if _builtin_exception_base(base, imports, scope_shadowed):\n continue\n", + " if _builtin_exception_base(base, imports, scope_shadowed):\n return \"method of an exception class\"\n"), + ("log_message context ignored", ' if (_http_request_handler_base(base, imports, scope_shadowed)\n and func.name == "log_message"):', " if False:"), + ("log_message context too broad", ' if (_http_request_handler_base(base, imports, scope_shadowed)\n and func.name == "log_message"):', + ' if _http_request_handler_base(base, imports, scope_shadowed):'), + ("module docstring not recorded", " docstring = bool(ast.get_docstring(ast.parse(files[path])))", + " docstring = True"), + ("class bodies not visited", " visit(child, classes + [child], scope_shadowed | _scope_bindings(child), child.body)", " pass"), + ("nested functions not visited", " visit(child, [], scope_shadowed | _scope_bindings(child), child.body)\n else:", + " pass\n else:"), + ("comments read as data", " if tok.type == tokenize.COMMENT:\n target = markers", + " if tok.type == tokenize.COMMENT:\n target = mentions"), + ("strings read as comments", " target = mentions\n else:\n continue", + " target = markers\n else:\n continue"), + ("marker words case-insensitive", r'MARKER = re.compile(r"\b(?:TODO|FIXME|XXX|TBD|STUB)\b|\b(?i:not implemented)\b")', + r'MARKER = re.compile(r"(?i)\b(?:TODO|FIXME|XXX|TBD|STUB)\b|\bnot implemented\b")'), + ("marker word boundary dropped", r'MARKER = re.compile(r"\b(?:TODO|FIXME|XXX|TBD|STUB)\b|\b(?i:not implemented)\b")', + r'MARKER = re.compile(r"(?:TODO|FIXME|XXX|TBD|STUB)|\b(?i:not implemented)\b")'), + ("quotes not tracked", " elif ch in \"\\\"'`\":\n quote = ch", + " elif False:\n quote = ch"), + ("string escapes ignored", ' if ch == "\\\\":\n i += 2\n', + " if False:\n i += 2\n"), + ("hash comment needs no whitespace", '(opener != "#" or i == 0 or line[i - 1].isspace())', "True"), + ("block comments not entered", " in_block, rest = True, rest[first + 2:]", + " in_block, rest = False, rest[first + 2:]"), + ("block comment end ignored", " in_block, rest = False, rest[end + 2:]", + " in_block, rest = True, rest[end + 2:]"), + ("mentions dropped", "for m in mentions)", "for m in [])"), + ("noop detection off", " if commands and all(c in NOOP_RUN for c in commands):", " if False:"), + ("block scalar not read", ' if not value or value[0] in "|>":', " if False:"), + ("block scalar ends late", " if nxt.strip() and len(nxt) - len(nxt.lstrip()) <= key_column:", + " if nxt.strip() and len(nxt) - len(nxt.lstrip()) < key_column:"), + ("inline comment kept", + " commands = [_strip_shell_comment(value).strip()]\n commands = [_strip_shell_comment(c).strip() for c in commands]", + " commands = [value.strip()]\n commands = [c.strip() for c in commands]"), + ("tests counted as tools", ' and area(p) == "tooling" and not p.endswith("__init__.py"))', + ' and True and not p.endswith("__init__.py"))'), + ("__init__ counted as tool", ' and area(p) == "tooling" and not p.endswith("__init__.py"))', + ' and area(p) == "tooling" and True)'), + ("basename ambiguity ignored", ' unique = {p: basenames[p.rsplit("/", 1)[-1]] == 1 for p in tools}', + " unique = {p: True for p in tools}"), + ("path reference not recognised", ' pattern = r"\\b" + re.escape(path) + r"\\b"\n', + ' pattern = r"\\bNOMATCH\\b"\n'), + ("import not recognised", " if isinstance(node, ast.Import):", " if False:"), + ("from-import list not recognised", " elif isinstance(node, ast.ImportFrom) and node.level == 0:", " elif False:"), + ("basename not recognised", ' r"|\\b" + re.escape(path.rsplit("/", 1)[-1]) + r"\\b")', + ' r"|\\bNOMATCH")'), + ("transitive exercise off", ' if other != path and report[other]["referenced"] and other in texts and path in tool_dependencies.get(other, set()):', + ' if False and other != path and report[other]["referenced"] and other in texts and path in tool_dependencies.get(other, set()):'), + ("site not production", 'PRODUCTION = {"tooling", "site"}', 'PRODUCTION = {"tooling"}'), + ("research counted as production", 'PRODUCTION = {"tooling", "site"}', 'PRODUCTION = {"tooling", "site", "research"}'), + ("vendor counted as site", ' if path.startswith("docs/site/vendor/"):\n return "vendor"', + ' if False:\n return "vendor"'), + ("undecodable decoded leniently", ' text = raw.decode("utf-8")', ' text = raw.decode("utf-8", "replace")'), + ("unparseable not a failure class", '"class": "unparseable"', '"class": "mention"'), + ("legitimate context still fails", '"class": "stub" if not stub["context"] else "stub-context"', '"class": "stub"'), + ("markers do not fail", 'f["class"] in {"stub", "marker", "noop", "undecodable", "unparseable"}]', + 'f["class"] in {"stub", "noop", "undecodable", "unparseable"}]'), + ("noop steps do not fail", 'f["class"] in {"stub", "marker", "noop", "undecodable", "unparseable"}]', + 'f["class"] in {"stub", "marker", "undecodable", "unparseable"}]'), + ("tool findings do not fail", '"status": "FAIL" if failing or failing_tools else "PASS"}', + '"status": "FAIL" if failing else "PASS"}'), + ("exit status always zero", ' return 1 if report["status"] == "FAIL" else 0\n', " return 0\n"), + ("unreadable revision exits zero", + ' print("G11_CENSUS_ERROR: cannot read revision %s: %s" % (args.rev, exc), file=sys.stderr)\n return 2\n', + ' print("G11_CENSUS_ERROR: cannot read revision %s: %s" % (args.rev, exc), file=sys.stderr)\n return 0\n'), +] + +RESULT = re.compile(r"^(FAILED|OK)\b(?: \(([^)]*)\))?\s*$", re.M) +SUMMARY = re.compile(r"Ran (\d+) tests? in [^\n]+") +FAIL_HEADER = re.compile(r"^FAIL: ([^(]+) \(([^)]+)\)(?: \[.*\])?(?: \(.*\))?$", re.M) +TRACE_FRAME = re.compile(r'File "([^"]*test_g11_census\.py)", line (\d+), in ([A-Za-z_]\w*)') +EXCEPTION = re.compile(r"(?m)^([A-Za-z_][A-Za-z0-9_]*(?:Error|Exception|Warning|Interrupt|Exit|StopIteration)):") + + +class HarnessError(Exception): + pass + + +TARGETS = { + "pass not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n pass\\n"), [(1, "a: pass")])'), + "ellipsis not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n ...\\n"), [(1, "a: ellipsis")])'), + "NotImplementedError not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n raise NotImplementedError\\n"), [(1, "a: raise NotImplementedError")])'), + "raise call not unwrapped": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n raise NotImplementedError(\'later\')\\n"),\n [(1, "a: raise NotImplementedError")])'), + "docstring-only not reported": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs(\'def a():\\n """Only a docstring."""\\n\'), [(1, "a: docstring only")])'), + "docstring not stripped": ("tests.test_g11_census.PythonStubs.test_docstring_then_stub_body", '[(1, "a: pass")])'), + "decorator context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), + "Protocol/ABC context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), + "exception context ignored": ("tests.test_g11_census.ReviewResidualControls.test_stub_exemptions_are_immediate_and_scope_safe", 'self.assertIn(name, stub_names)'), + "log_message context ignored": ("tests.test_g11_census.PythonStubs.test_request_handler_log_message_override_is_legitimate", "self.assertEqual(found,"), + "log_message context too broad": ("tests.test_g11_census.PythonStubs.test_request_handler_log_message_override_is_legitimate", "self.assertEqual(found,"), + "module docstring not recorded": ("tests.test_g11_census.ToolChecks.test_module_docstring_is_recorded", "self.assertEqual((report["), + "class bodies not visited": ("tests.test_g11_census.PythonStubs.test_methods_async_and_nested", "self.assertEqual(stubs(text),"), + "nested functions not visited": ("tests.test_g11_census.PythonStubs.test_methods_async_and_nested", "self.assertEqual(stubs(text),"), + "comments read as data": ("tests.test_g11_census.Markers.test_python_comment_is_marker_and_string_is_mention", "self.assertEqual(classes(run("), + "strings read as comments": ("tests.test_g11_census.Markers.test_python_comment_is_marker_and_string_is_mention", "self.assertEqual(classes(run("), + "marker words case-insensitive": ("tests.test_g11_census.Markers.test_lowercase_words_and_lookalikes_are_not_markers", "self.assertEqual(classes(run("), + "marker word boundary dropped": ("tests.test_g11_census.Markers.test_lowercase_words_and_lookalikes_are_not_markers", "self.assertEqual(classes(run("), + "quotes not tracked": ("tests.test_g11_census.Markers.test_javascript_comments_and_strings", "self.assertEqual(classes(run("), + "string escapes ignored": ("tests.test_g11_census.Markers.test_escaped_quote_inside_string", "self.assertEqual(classes(run("), + "hash comment needs no whitespace": ("tests.test_g11_census.Markers.test_shell_comments_and_strings", "self.assertEqual(classes(run("), + "block comments not entered": ("tests.test_g11_census.Markers.test_javascript_comments_and_strings", "self.assertEqual(classes(run("), + "block comment end ignored": ("tests.test_g11_census.Markers.test_javascript_comments_and_strings", "self.assertEqual(classes(run("), + "mentions dropped": ("tests.test_g11_census.Markers.test_python_comment_is_marker_and_string_is_mention", "self.assertEqual(classes(run("), + "noop detection off": ("tests.test_g11_census.Workflows.test_noop_steps", "self.assertEqual(found,"), + "block scalar not read": ("tests.test_g11_census.Workflows.test_noop_steps", "self.assertEqual(found,"), + "block scalar ends late": ("tests.test_g11_census.Workflows.test_noop_steps", "self.assertEqual(found,"), + "inline comment kept": ("tests.test_g11_census.Workflows.test_noop_steps", "self.assertEqual(found,"), + "tests counted as tools": ("tests.test_g11_census.ToolChecks.test_tests_and_init_files_are_not_tools", "self.assertEqual(sorted(report),"), + "__init__ counted as tool": ("tests.test_g11_census.ToolChecks.test_tests_and_init_files_are_not_tools", "self.assertEqual(sorted(report),"), + "basename ambiguity ignored": ("tests.test_g11_census.ToolChecks.test_ambiguous_basename_needs_the_full_path", 'self.assertFalse(ambiguous["tools/run.py"]["documented"])'), + "path reference not recognised": ("tests.test_g11_census.ToolChecks.test_full_path_counts_even_when_basename_is_ambiguous", 'self.assertTrue(report["tools/run.py"]["documented"])'), + "import not recognised": ("tests.test_g11_census.ToolChecks.test_documented_and_exercised_by_path_import_and_basename", 'self.assertTrue(report[path]["referenced"])'), + "from-import list not recognised": ("tests.test_g11_census.ToolChecks.test_documented_and_exercised_by_path_import_and_basename", 'self.assertTrue(report[path]["referenced"])'), + "basename not recognised": ("tests.test_g11_census.ToolChecks.test_documented_and_exercised_by_path_import_and_basename", 'self.assertTrue(report[path]["documented"])'), + "transitive exercise off": ("tests.test_g11_census.ToolChecks.test_exercised_through_an_exercised_tool", 'self.assertEqual(report["tools/helper.py"]["referenced_by"],'), + "site not production": ("tests.test_g11_census.Markers.test_site_marker_fails_and_vendor_marker_does_not", 'self.assertEqual(run({"docs/site/app.mjs":'), + "research counted as production": ("tests.test_g11_census.PythonStubs.test_production_stub_fails_but_research_stub_does_not", 'self.assertEqual(research["status"],'), + "vendor counted as site": ("tests.test_g11_census.Markers.test_site_marker_fails_and_vendor_marker_does_not", 'self.assertEqual(run({"docs/site/vendor/lib.js":'), + "undecodable decoded leniently": ("tests.test_g11_census.FailClosed.test_undecodable_and_unparseable_production_files_fail", 'self.assertEqual([f["class"] for f in report["failing"]], [expected])'), + "unparseable not a failure class": ("tests.test_g11_census.FailClosed.test_undecodable_and_unparseable_production_files_fail", 'self.assertEqual(report["status"], "FAIL")'), + "legitimate context still fails": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), + "markers do not fail": ("tests.test_g11_census.Markers.test_production_marker_fails", 'self.assertEqual(report["status"], "FAIL")'), + "noop steps do not fail": ("tests.test_g11_census.Workflows.test_production_noop_fails", 'self.assertEqual(run({".github/workflows/w.yml": " - run: true'), + "tool findings do not fail": ("tests.test_g11_census.ToolChecks.test_undocumented_and_unexercised_tools_fail", 'self.assertEqual(result["status"], "FAIL")'), + "exit status always zero": ("tests.test_g11_census.CommandLine.test_exit_codes_and_exact_committed_bytes", "self.assertEqual(result.returncode, 1)"), + "unreadable revision exits zero": ("tests.test_g11_census.CommandLine.test_exit_codes_and_exact_committed_bytes", 'self.assertEqual(self.cli(root, "--rev", "no-such-revision").returncode, 2)'), +} + + +NEW_MUTATIONS = [ + ("decoy decorator names accepted", "census", + " if _trusted_decorator(dec, imports, scope_shadowed):\n resolved = _resolve_scoped_import(dec, imports, scope_shadowed)", + ' if _decorator_name(dec) in LEGIT_DECORATORS:\n resolved = ("abc", "abstractmethod") if _decorator_name(dec) == "abstractmethod" else ("typing", "overload")', + "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", 'self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]),'), + ("decoy abstract-base names accepted", "census", ' if base_kind == "Protocol":', " if _base_name(base) in LEGIT_BASES:", + "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", 'self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]),'), + ("decoy request-handler suffix accepted", "census", ' if (_http_request_handler_base(base, imports, scope_shadowed)\n and func.name == "log_message"):', + ' if _base_name(base).endswith("RequestHandler") and func.name == "log_message":', + "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", 'self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]),'), + ("comment and string mentions count as exercise", "census", + 'def python_tool_references(text, tools, unique, mode="script"):\n references = set()', + 'def python_tool_references(text, tools, unique, mode="script"):\n references = {p for p in tools if p in text}', + "tests.test_g11_census.AdversarialControls.test_comment_and_string_mentions_do_not_exercise_tools", + 'self.assertFalse(report["tools"]["tools/runner.py"]["referenced"])'), + ("empty workflow run block not detected", "census", " if not commands:", " if False:", + "tests.test_g11_census.AdversarialControls.test_empty_and_comment_only_run_blocks_fail", "self.assertEqual(found,"), + ("YAML comments omitted", "census", ' elif path.endswith((".yml", ".yaml")):', " elif False:", + "tests.test_g11_census.AdversarialControls.test_yaml_comments_are_scanned_for_markers", "self.assertEqual(classes(report),"), + ("mutation attribution accepts wrong assertion", "harness", ' and outcome.get("traceback_line") == expected_line\n and outcome.get("exception_type")', + ' and True\n and outcome.get("exception_type")', + "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", + "self.assertFalse(g11m.qualifies_kill(wrong_line, expected, 17))"), + ("function-local decorator shadow accepted", "census", + " if binding and binding in scope_shadowed:\n return None", + " if False:\n return None", + "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", + 'self.assertEqual([(f["class"], f["detail"]) for f in local_report["findings"]'), + ("python -c path mention counts as execution", "census", + ' if "-c" in tokens[1:]:\n continue', + " if False:\n continue", + "tests.test_g11_census.AdversarialControls.test_comment_and_string_mentions_do_not_exercise_tools", + 'self.assertFalse(python_cmd["tools"]["tools/runner.py"]["referenced"])'), + ("shell -c path mention counts as execution", "census", + ' if "-c" not in tokens[1:]:', + " if True:", + "tests.test_g11_census.AdversarialControls.test_comment_and_string_mentions_do_not_exercise_tools", + 'self.assertFalse(shell_cmd["tools"]["tools/runner.py"]["referenced"])'), + ("template literal import text counts as static import", "census", + ' if quote == "`":\n if ch == "\\\\" and i + 1 < len(text):\n out.extend((" ", " "))\n i += 2\n continue\n if ch == quote:\n quote = None\n out.append("\\n" if ch == "\\n" else " ")\n i += 1\n continue', + ' if quote == "`":\n if ch == "\\\\" and i + 1 < len(text):\n out.extend((" ", " "))\n i += 2\n continue\n if ch == quote:\n quote = None\n out.append(ch)\n i += 1\n continue', + "tests.test_g11_census.AdversarialControls.test_comment_and_string_mentions_do_not_exercise_tools", + 'self.assertFalse(js_cmd["tools"]["tools/runner.mjs"]["referenced"])'), + ("parser drops method identity", "harness", + ' test_id += "." + method\n result["test_id"] = test_id', + ' pass\n result["test_id"] = test_id', + "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", + 'self.assertEqual(parsed["test_id"], expected_parser)'), + ("parser repeats qualified method identity", "harness", + ' if not test_id.endswith("." + method):\n test_id += "." + method', + ' if True:\n test_id += "." + method', + "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", + 'self.assertEqual(parsed_current["test_id"], expected_parser)'), + ("subTest failure header not parsed", "harness", + 'FAIL_HEADER = re.compile(r"^FAIL: ([^(]+) \\(([^)]+)\\)(?: \\[.*\\])?(?: \\(.*\\))?$", re.M)\nTRACE_FRAME', + 'FAIL_HEADER = re.compile(r"^FAIL: ([^(]+) \\(([^)]+)\\)$", re.M)\nTRACE_FRAME', + "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", + 'self.assertEqual(parsed_subtest["test_id"], expected_parser)'), + ("wrong test id credited", "harness", + ' and outcome.get("test_id") == expected_test\n and outcome.get("traceback_method")', + ' and True\n and outcome.get("traceback_method")', + "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", + "self.assertFalse(g11m.qualifies_kill(wrong_test, expected, 17))"), + ("nonfailure status credited", "harness", + ' and outcome.get("status") == "FAILED"\n and outcome.get("exit")', + ' and True\n and outcome.get("exit")', + "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", + "self.assertFalse(g11m.qualifies_kill(not_failed, expected, 17))"), + ("error outcome credited", "harness", + ' and outcome.get("failures") == 1\n and outcome.get("errors") == 0', + ' and outcome.get("failures") == 1\n and True', + "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", + "self.assertFalse(g11m.qualifies_kill(has_errors, expected, 17))"), + ("skipped outcome credited", "harness", + ' and outcome.get("skipped") == 0\n and outcome.get("test_id")', + ' and True\n and outcome.get("test_id")', + "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", + "self.assertFalse(g11m.qualifies_kill(has_skips, expected, 17))"), + ("quoted workflow command boundary ignored", "census", + " value = value[1:-1]", + " value = value", + "tests.test_g11_census.AdversarialControls.test_comment_and_string_mentions_do_not_exercise_tools", + "self.assertEqual(g11._workflow_commands(workflow_text)[0][1], [\"sh -c 'echo tools/runner.py'\"])"), + ("FAIL header mistaken for exception", "harness", + 'EXCEPTION = re.compile(r"(?m)^([A-Za-z_][A-Za-z0-9_]*(?:Error|Exception|Warning|Interrupt|Exit|StopIteration)):")\n\n\nclass HarnessError', + 'EXCEPTION = re.compile(r"(?m)^([A-Za-z_][A-Za-z0-9_]*):")\n\n\nclass HarnessError', + "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", + "self.assertTrue(g11m.qualifies_kill(parsed, expected_parser, 123))"), + ("outer class exemption leaks", "census", + " for cls in classes[-1:]:", " for cls in reversed(classes):", + "tests.test_g11_census.ReviewResidualControls.test_stub_exemptions_are_immediate_and_scope_safe", + "self.assertIn(name, stub_names)"), + ("local base shadow accepted", "census", + " if binding and binding in scope_shadowed:\n return None", + " if False:\n return None", + "tests.test_g11_census.ReviewResidualControls.test_stub_exemptions_are_immediate_and_scope_safe", + 'self.assertIn("shadowed_abc", stub_names)'), + ("uncalled helper counts as exercise", "census", + " if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):\n # A definition is not a call: only decorators/defaults execute here.", + " if False:\n # A definition is not a call: only decorators/defaults execute here.", + "tests.test_g11_census.ReviewResidualControls.test_tool_reachability_and_heredoc_boundaries", + 'self.assertFalse(uncalled["tools"]["tools/runner.py"]["referenced"])'), + ("heredoc body counted as command", "census", + " if heredoc:", " if False:", + "tests.test_g11_census.ReviewResidualControls.test_tool_reachability_and_heredoc_boundaries", + 'self.assertFalse(heredoc["tools"]["tools/runner.py"]["referenced"])'), + ("mutation inventory shrink accepted", "harness", + "\n if tuple(cases) != declared:\n raise HarnessError(\"mutation inventory mismatch: complete case tuples differ\")\n", "\n if False:\n raise HarnessError(\"mutation inventory mismatch: complete case tuples differ\")\n", + "tests.test_g11_census.ReviewResidualControls.test_mutation_inventory_is_sealed", + 'self.assertRaisesRegex(g11m.HarnessError, "mutation inventory")'), + ("hostile Python environment survives", "harness", + " for name in PYTHON_ENVIRONMENT_KEYS:\n env.pop(name, None)", " for name in ():\n env.pop(name, None)", + "tests.test_g11_census.ReviewResidualControls.test_mutation_child_environment_is_sanitized", + 'self.assertEqual(outcome.get("status"), "OK")'), + ("in-scope tracked symlink silently skipped", "census", + " if path.endswith(SCANNED) and (kind != b\"blob\" or mode == b\"120000\"):", + " if False:", + "tests.test_g11_census.ReviewResidualControls.test_in_scope_tracked_symlink_is_rejected", + 'self.assertRaisesRegex(ValueError, "in-scope non-regular tracked path")'), + ("decoy TestCase base credited", "census", + 'and any(_resolve_import(base, imports) == ("unittest", "TestCase")', + 'and any(_base_name(base) == "TestCase"', + "tests.test_g11_census.ReviewResidualControls.test_tool_reachability_and_heredoc_boundaries", + 'self.assertFalse(decoy["tools"]["tools/runner.py"]["referenced"])'), + +] +EXPECTED_MUTATION_NAMES = ( + "pass not detected", + "ellipsis not detected", + "NotImplementedError not detected", + "raise call not unwrapped", + "docstring-only not reported", + "docstring not stripped", + "decorator context ignored", + "Protocol/ABC context ignored", + "exception context ignored", + "log_message context ignored", + "log_message context too broad", + "module docstring not recorded", + "class bodies not visited", + "nested functions not visited", + "comments read as data", + "strings read as comments", + "marker words case-insensitive", + "marker word boundary dropped", + "quotes not tracked", + "string escapes ignored", + "hash comment needs no whitespace", + "block comments not entered", + "block comment end ignored", + "mentions dropped", + "noop detection off", + "block scalar not read", + "block scalar ends late", + "inline comment kept", + "tests counted as tools", + "__init__ counted as tool", + "basename ambiguity ignored", + "path reference not recognised", + "import not recognised", + "from-import list not recognised", + "basename not recognised", + "transitive exercise off", + "site not production", + "research counted as production", + "vendor counted as site", + "undecodable decoded leniently", + "unparseable not a failure class", + "legitimate context still fails", + "markers do not fail", + "noop steps do not fail", + "tool findings do not fail", + "exit status always zero", + "unreadable revision exits zero", + "decoy decorator names accepted", + "decoy abstract-base names accepted", + "decoy request-handler suffix accepted", + "comment and string mentions count as exercise", + "empty workflow run block not detected", + "YAML comments omitted", + "mutation attribution accepts wrong assertion", + "function-local decorator shadow accepted", + "python -c path mention counts as execution", + "shell -c path mention counts as execution", + "template literal import text counts as static import", + "parser drops method identity", + "parser repeats qualified method identity", + "subTest failure header not parsed", + "wrong test id credited", + "nonfailure status credited", + "error outcome credited", + "skipped outcome credited", + "quoted workflow command boundary ignored", + "FAIL header mistaken for exception", + "outer class exemption leaks", + "local base shadow accepted", + "uncalled helper counts as exercise", + "heredoc body counted as command", + "mutation inventory shrink accepted", + "hostile Python environment survives", + "in-scope tracked symlink silently skipped", + "decoy TestCase base credited", +) + + +def _build_cases(): + cases = [] + for name, old, new in MUTATIONS: + target_test, fragment = TARGETS[name] + cases.append((name, "census", old, new, target_test, fragment)) + cases.extend(NEW_MUTATIONS) + return cases + + +def _declared_mutation_cases(source): + tree = ast.parse(source) + required = {"MUTATIONS", "TARGETS", "NEW_MUTATIONS", "EXPECTED_MUTATION_NAMES"} + values = {} + for node in tree.body: + if (isinstance(node, ast.Assign) and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id in required): + values[node.targets[0].id] = ast.literal_eval(node.value) + if set(values) != required: + raise HarnessError("mutation inventory declarations are incomplete") + cases = [] + for item in values["MUTATIONS"]: + if not isinstance(item, (tuple, list)) or len(item) != 3: + raise HarnessError("mutation inventory declaration has an invalid census tuple") + name, old, new = item + try: + target_test, fragment = values["TARGETS"][name] + except (KeyError, TypeError, ValueError) as exc: + raise HarnessError("mutation inventory target missing for %r: %s" % (name, exc)) + cases.append((name, "census", old, new, target_test, fragment)) + cases.extend(values["NEW_MUTATIONS"]) + expected_names = tuple(values["EXPECTED_MUTATION_NAMES"]) + if tuple(case[0] for case in cases) != expected_names: + raise HarnessError("mutation inventory names disagree with literal declarations") + return tuple(cases) + + +def _validate_mutation_inventory(cases): + try: + declared = _declared_mutation_cases(Path(__file__).read_text(encoding="utf-8")) + except (OSError, SyntaxError, ValueError, TypeError, KeyError) as exc: + raise HarnessError("mutation inventory cannot be parsed: %s" % exc) + if tuple(cases) != declared: + raise HarnessError("mutation inventory mismatch: complete case tuples differ") + + + +def _assertion_line(test_source, test_id, fragment): + parts = test_id.split(".") + class_name, method_name = parts[-2], parts[-1] + tree = ast.parse(test_source) + class_node = next((n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == class_name), None) + if class_node is None: + raise HarnessError("target class missing: " + class_name) + method = next((n for n in class_node.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) + and n.name == method_name), None) + if method is None: + raise HarnessError("target test missing: " + test_id) + matches = [] + for node in ast.walk(method): + if isinstance(node, ast.Call): + segment = ast.get_source_segment(test_source, node) or "" + if fragment in segment: + matches.append(node.lineno) + if len(matches) != 1: + raise HarnessError("assertion fragment %r matched %d calls in %s" % (fragment, len(matches), test_id)) + return matches[0] + + +def _parse_run(proc): + output = proc.stdout + "\n" + proc.stderr + summary = SUMMARY.search(output) + ran = int(summary.group(1)) if summary else -1 + result = {"exit": proc.returncode, "ran": ran, "failures": 0, "errors": 0, "skipped": 0, + "test_id": None, "traceback_method": None, "traceback_line": None, + "exception_type": None, "parsed": bool(summary)} + status = RESULT.findall(output) + if status: + result["status"] = status[-1][0] + detail = status[-1][1] or "" + counts = dict(re.findall(r"(failures|errors|skipped)=(\d+)", detail)) + result["failures"] = int(counts.get("failures", 0)) + result["errors"] = int(counts.get("errors", 0)) + result["skipped"] = int(counts.get("skipped", 0)) + header = FAIL_HEADER.search(output) + if header: + method, test_id = header.group(1), header.group(2) + # Python 3.11+ prints "method (module.Class.method)"; earlier versions omit the method. + if not test_id.endswith("." + method): + test_id += "." + method + result["test_id"] = test_id + frames = TRACE_FRAME.findall(output) + if frames: + _path, line, method = frames[-1] + result["traceback_line"] = int(line) + result["traceback_method"] = method + exc = EXCEPTION.search(output) + if exc: + result["exception_type"] = exc.group(1) + return result + + +def qualifies_kill(outcome, expected_test, expected_line): + expected_method = expected_test.rsplit(".", 1)[-1] + return ( + outcome.get("parsed") is True + and outcome.get("status") == "FAILED" + and outcome.get("exit") == 1 + and outcome.get("ran") == 1 + and outcome.get("failures") == 1 + and outcome.get("errors") == 0 + and outcome.get("skipped") == 0 + and outcome.get("test_id") == expected_test + and outcome.get("traceback_method") == expected_method + and outcome.get("traceback_line") == expected_line + and outcome.get("exception_type") == "AssertionError" + ) + + +PYTHON_ENVIRONMENT_KEYS = ("PYTHONPATH", "PYTHONHOME", "PYTHONPYCACHEPREFIX", + "PYTHONSTARTUP", "PYTHONUSERBASE") + + +def _clean_child_env(source=None): + env = dict(os.environ if source is None else source) + for name in PYTHON_ENVIRONMENT_KEYS: + env.pop(name, None) + env["PYTHONNOUSERSITE"] = "1" + return env + + +def _run(workdir, test_id=None, env=None): + command = [sys.executable, "-B", "-S"] + if sys.flags.optimize: + command.append("-O") + command.extend(["-m", "unittest"]) + command.append(test_id if test_id else "tests.test_g11_census") + command.append("-v") + proc = subprocess.run(command, cwd=workdir, env=_clean_child_env(env), + capture_output=True, text=True, timeout=300) + return _parse_run(proc) + + +def _sandbox(tool_source, test_source, harness_source, workdir): + os.makedirs(os.path.join(workdir, "tests"), exist_ok=True) + os.makedirs(os.path.join(workdir, "tools", "cbz"), exist_ok=True) + Path(workdir, TESTS).write_text(test_source, encoding="utf-8") + Path(workdir, TOOL).write_text(tool_source, encoding="utf-8") + Path(workdir, HARNESS).write_text(harness_source, encoding="utf-8") + + +def run_selftest(root): + tool_path = os.path.join(root, TOOL) + test_path = os.path.join(root, TESTS) + harness_path = os.path.join(root, HARNESS) + with open(tool_path, encoding="utf-8") as fh: + original_tool = fh.read() + with open(test_path, encoding="utf-8") as fh: + test_source = fh.read() + with open(harness_path, encoding="utf-8") as fh: + original_harness = fh.read() + cases = _build_cases() + _validate_mutation_inventory(cases) + for name, target, old, new, test_id, fragment in cases: + source = original_tool if target == "census" else original_harness + if source.count(old) != 1 or old == new: + raise HarnessError("mutation %r: target occurs %d times in %s (expected exactly 1)" % + (name, source.count(old), target)) + _assertion_line(test_source, test_id, fragment) + baseline = None + results = [] + with tempfile.TemporaryDirectory(prefix="g11-mut-") as tmp: + base = os.path.join(tmp, "baseline") + _sandbox(original_tool, test_source, original_harness, base) + baseline = _run(base) + baseline_ok = (baseline.get("parsed") and baseline.get("exit") == 0 and baseline.get("ran") == 38 + and baseline.get("failures") == 0 and baseline.get("errors") == 0 + and baseline.get("skipped") == 0 and baseline.get("status") == "OK") + if baseline_ok: + for index, (name, target, old, new, test_id, fragment) in enumerate(cases): + work = os.path.join(tmp, "m%02d" % index) + tool_source = original_tool + harness_source = original_harness + if target == "census": + tool_source = original_tool.replace(old, new, 1) + else: + harness_source = original_harness.replace(old, new, 1) + _sandbox(tool_source, test_source, harness_source, work) + outcome = _run(work, test_id) + expected_line = _assertion_line(test_source, test_id, fragment) + results.append(dict(outcome, mutation=name, target=target, expected_test=test_id, + expected_line=expected_line, + killed=qualifies_kill(outcome, test_id, expected_line))) + killed = sum(r["killed"] for r in results) + return {"baseline": baseline, "mutations": results, "killed": killed, "total": len(cases), + "executed": len(results), + "status": "PASS" if baseline_ok and len(results) == len(cases) and killed == len(results) else "FAIL"} + + +def main(argv=None): + parser = argparse.ArgumentParser(description="Mutation self-test of the G11 census and its kill-attribution guard.") + parser.add_argument("--root", default=".") + parser.add_argument("--json", action="store_true") + args = parser.parse_args(argv) + try: + report = run_selftest(os.path.abspath(args.root)) + except (HarnessError, OSError, subprocess.SubprocessError) as exc: + print("G11_SELFTEST_HARNESS_ERROR: %s" % exc, file=sys.stderr) + return 2 + if args.json: + print(json.dumps(report, indent=1, sort_keys=True)) + else: + for r in report["mutations"]: + print("%-8s %s (target %s, exit %d, failures %d, errors %d, skips %d, line %s/%s)" % + ("KILLED" if r["killed"] else "SURVIVED", r["mutation"], r["target"], r["exit"], + r["failures"], r["errors"], r["skipped"], r["traceback_line"], r["expected_line"])) + print("G11_SELFTEST_%s: baseline %d tests, %d/%d mutations killed (%d executed)" % + (report["status"], report["baseline"].get("ran", -1), report["killed"], + report["total"], report["executed"])) + return 0 if report["status"] == "PASS" else 1 + + +if __name__ == "__main__": + sys.exit(main())