From 81c27f23cef9e4fd03ea35027ffd1c495c53aa07 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:31:00 -0500 Subject: [PATCH 01/27] CBZ-0 G11: bind exemptions and exercise to parsed source --- tools/cbz/g11_census.py | 765 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 765 insertions(+) create mode 100644 tools/cbz/g11_census.py diff --git a/tools/cbz/g11_census.py b/tools/cbz/g11_census.py new file mode 100644 index 000000000..ee2f2bc05 --- /dev/null +++ b/tools/cbz/g11_census.py @@ -0,0 +1,765 @@ +#!/usr/bin/env python3 +"""CBZ-0 gate G11 census: stub scan and tool documentation check (stdlib only). + +Reads the exact blob bytes of one git revision and reports: + + stub a Python function whose body, after any docstring, is only `pass`, + `...` or `raise NotImplementedError`, or is the docstring alone. + A stub under @abstractmethod or @overload, or in a Protocol, ABC or + exception class, is reported with that context and does not fail. + marker TODO, FIXME, XXX, TBD or STUB (upper case) or "not implemented" in + a code comment. In Python the comment is found by tokenizing; in + .mjs/.js/.cjs/.sh a comment is found by a quote-parity heuristic. + The same words in string data are recorded as mentions only. + noop a workflow step whose `run:` does nothing: `true`, `:` or `exit 0`. + tools every executable source under tools/ (not tests, not __init__.py), + with the tracked Markdown files that name it and the evidence that + something exercises it: a test or workflow naming or importing it, + or an import from a tool that is itself exercised. + +Every path has an area. Only `tooling` (tools/, .github/) and `site` +(docs/site/ outside vendor/) are production; findings elsewhere are reported +and never fail. A production file that is not UTF-8, or Python that does not +parse, fails closed. + +Exit status: 0 when nothing in production fails, 1 when something does, 2 when +the revision cannot be read. The census is evidence for a reviewer, never a +G11 certificate. +""" +from __future__ import annotations + +import argparse +import ast +import builtins +import posixpath +import shlex +import io +import json +import re +import subprocess +import sys +import tokenize + +MARKER = re.compile(r"\b(?:TODO|FIXME|XXX|TBD|STUB)\b|\b(?i:not implemented)\b") +CODE_SUFFIXES = (".py", ".mjs", ".js", ".cjs", ".sh") +SCANNED = CODE_SUFFIXES + (".md", ".yml", ".yaml") +LEGIT_DECORATORS = {"abstractmethod", "overload"} +LEGIT_BASES = {"Protocol", "ABC"} +NOOP_RUN = {"true", ":", "exit 0"} +PRODUCTION = {"tooling", "site"} + + +def area(path): + name = path.rsplit("/", 1)[-1] + if (path.startswith("tests/") or "/tests/" in path or name == "conftest.py" + or name.startswith("test_") or name.endswith("_test.py") or ".test." in name): + return "tests" + if path.startswith("tools/") or path.startswith(".github/"): + return "tooling" + if path.startswith("docs/site/vendor/"): + return "vendor" + if path.startswith("docs/site/"): + return "site" + if path.startswith("experiments/"): + return "research" + return "evidence" + + +def _decorator_name(node): + while isinstance(node, ast.Call): + node = node.func + if isinstance(node, ast.Attribute): + return node.attr + if isinstance(node, ast.Name): + return node.id + return "" + + +def _base_name(node): + if isinstance(node, ast.Subscript): + node = node.value + if isinstance(node, ast.Attribute): + return node.attr + if isinstance(node, ast.Name): + return node.id + return "" + +def _dotted_name(node): + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + base = _dotted_name(node.value) + return base + "." + node.attr if base else "" + return "" + + +def _target_names(node): + if isinstance(node, ast.Name): + return {node.id} + if isinstance(node, (ast.Tuple, ast.List)): + names = set() + for item in node.elts: + names.update(_target_names(item)) + return names + if isinstance(node, ast.Starred): + return _target_names(node.value) + return set() + + +def _import_bindings(tree): + modules, symbols, shadowed = {}, {}, set() + for node in tree.body: + if type(node) is ast.Import: + for alias in node.names: + bound = alias.asname or alias.name.split(".", 1)[0] + modules[bound] = alias.name if alias.asname else alias.name.split(".", 1)[0] + elif type(node) is ast.ImportFrom and node.level == 0: + for alias in node.names: + symbols[alias.asname or alias.name] = (node.module or "", alias.name) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + shadowed.add(node.name) + elif isinstance(node, ast.Assign): + for target in node.targets: + shadowed.update(_target_names(target)) + elif isinstance(node, (ast.AnnAssign, ast.AugAssign)): + shadowed.update(_target_names(node.target)) + for name in shadowed: + modules.pop(name, None) + symbols.pop(name, None) + return {"modules": modules, "symbols": symbols, "shadowed": shadowed} + + +def _resolve_import(node, imports): + if isinstance(node, ast.Name): + if node.id in imports["shadowed"]: + return None + return imports["symbols"].get(node.id) + dotted = _dotted_name(node) + if not dotted: + return None + parts = dotted.split(".") + if parts[0] in imports["shadowed"]: + return None + module = imports["modules"].get(parts[0]) + if module is None or len(parts) < 2: + return None + resolved = module + "." + ".".join(parts[1:]) + return tuple(resolved.rsplit(".", 1)) + + +def _trusted_decorator(node, imports): + resolved = _resolve_import(node, imports) + return resolved in {("abc", "abstractmethod"), ("typing", "overload"), + ("typing_extensions", "overload")} + + +def _trusted_abstract_base(node, imports): + resolved = _resolve_import(node, imports) + return resolved in {("abc", "ABC"), ("typing", "Protocol"), + ("typing_extensions", "Protocol")} + + +def _builtin_exception_base(node, imports): + resolved = _resolve_import(node, imports) + name = None + if resolved and resolved[0] == "builtins": + name = resolved[1] + elif isinstance(node, ast.Name) and node.id in builtins.__dict__ and node.id not in imports["shadowed"]: + name = node.id + if not name: + return False + candidate = getattr(builtins, name, None) + return isinstance(candidate, type) and issubclass(candidate, BaseException) + + +def _http_request_handler_base(node, imports): + return _resolve_import(node, imports) in { + ("http.server", "BaseHTTPRequestHandler"), + ("http.server", "SimpleHTTPRequestHandler"), + ("http.server", "CGIHTTPRequestHandler"), + } + + + + +def _stub_kind(body): + rest = body + if rest and isinstance(rest[0], ast.Expr) and isinstance(rest[0].value, ast.Constant) \ + and isinstance(rest[0].value.value, str): + rest = rest[1:] + if not rest: + return "docstring only" + if len(rest) != 1: + return None + stmt = rest[0] + if isinstance(stmt, ast.Pass): + return "pass" + if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Constant) and stmt.value.value is Ellipsis: + return "ellipsis" + if isinstance(stmt, ast.Raise) and stmt.exc is not None: + exc = stmt.exc.func if isinstance(stmt.exc, ast.Call) else stmt.exc + if isinstance(exc, ast.Name) and exc.id == "NotImplementedError": + return "raise NotImplementedError" + return None + + +def _legit_context(func, classes, imports): + for dec in func.decorator_list: + if _trusted_decorator(dec, imports): + return "@" + _decorator_name(dec) + for cls in reversed(classes): + for base in cls.bases: + name = _base_name(base) + if _trusted_abstract_base(base, imports): + return "method of a %s class" % name + if _builtin_exception_base(base, imports): + return "method of an exception class" + if _http_request_handler_base(base, imports) and func.name == "log_message": + return "log_message override that silences http.server request logging" + return None + + +def python_stubs(tree, imports=None): + found = [] + if imports is None: + imports = _import_bindings(tree) + + def visit(node, classes): + for child in ast.iter_child_nodes(node): + if isinstance(child, ast.ClassDef): + visit(child, classes + [child]) + elif isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): + kind = _stub_kind(child.body) + if kind: + found.append({"line": child.lineno, "name": child.name, "kind": kind, + "context": _legit_context(child, classes, imports)}) + visit(child, []) + else: + visit(child, classes) + visit(tree, []) + return found + + +def python_words(raw): + markers, mentions = [], [] + for tok in tokenize.tokenize(io.BytesIO(raw).readline): + if tok.type == tokenize.COMMENT: + target = markers + elif tok.type == tokenize.STRING or tok.type == getattr(tokenize, "FSTRING_MIDDLE", -1): + target = mentions + else: + continue + for m in MARKER.finditer(tok.string): + target.append({"line": tok.start[0] + tok.string[:m.start()].count("\n"), "word": m.group(0)}) + return markers, mentions + + +def _comment_start(line, opener): + # Index where a comment opens outside quotes, by quote parity; -1 if none. + quote = None + i = 0 + while i < len(line): + ch = line[i] + if quote: + if ch == "\\": + i += 2 + continue + if ch == quote: + quote = None + elif ch in "\"'`": + quote = ch + elif line.startswith(opener, i) and (opener != "#" or i == 0 or line[i - 1].isspace()): + return i + i += 1 + return -1 + + +def text_words(text, path): + markers, mentions = [], [] + opener = "#" if path.endswith(".sh") else "//" + in_block = False + for number, line in enumerate(text.splitlines(), 1): + comment_parts, data_parts = [], [] + if opener == "//": + rest = line + while rest: + if in_block: + end = rest.find("*/") + comment_parts.append(rest if end < 0 else rest[:end]) + if end < 0: + rest = "" + else: + in_block, rest = False, rest[end + 2:] + continue + line_at, block_at = _comment_start(rest, "//"), _comment_start(rest, "/*") + starts = [p for p in (line_at, block_at) if p >= 0] + if not starts: + data_parts.append(rest) + break + first = min(starts) + data_parts.append(rest[:first]) + if first == line_at: + comment_parts.append(rest[first:]) + break + in_block, rest = True, rest[first + 2:] + else: + at = _comment_start(line, "#") + data_parts.append(line if at < 0 else line[:at]) + if at >= 0: + comment_parts.append(line[at:]) + for part in comment_parts: + markers.extend({"line": number, "word": m.group(0)} for m in MARKER.finditer(part)) + for part in data_parts: + mentions.extend({"line": number, "word": m.group(0)} for m in MARKER.finditer(part)) + return markers, mentions + + + +def _yaml_comment_start(line): + quote = None + i = 0 + while i < len(line): + ch = line[i] + if quote: + if quote == '"' and ch == "\\": + i += 2 + continue + if quote == "'" and ch == "'" and i + 1 < len(line) and line[i + 1] == "'": + i += 2 + continue + if ch == quote: + quote = None + elif ch in "\"'": + quote = ch + elif ch == "#" and (i == 0 or line[i - 1].isspace()): + return i + i += 1 + return -1 + + +def yaml_words(text): + markers, mentions = [], [] + for number, line in enumerate(text.splitlines(), 1): + at = _yaml_comment_start(line) + data = line if at < 0 else line[:at] + comment = "" if at < 0 else line[at + 1:] + mentions.extend({"line": number, "word": m.group(0)} for m in MARKER.finditer(data)) + markers.extend({"line": number, "word": m.group(0)} for m in MARKER.finditer(comment)) + return markers, mentions + + +def _strip_shell_comment(line): + at = _comment_start(line, "#") + return line if at < 0 else line[:at] + + +def _workflow_commands(text): + found, lines = [], text.splitlines() + for number, line in enumerate(lines, 1): + m = re.match(r"^(\s*)(?:-\s+)?run:\s*(.*?)\s*$", line) + if not m: + continue + value = m.group(2).strip() + if value.startswith("#"): + value = "" + if not value or value[0] in "|>": + key_column = line.index("run:") + body = [] + for nxt in lines[number:]: + if nxt.strip() and len(nxt) - len(nxt.lstrip()) <= key_column: + break + body.append(nxt.strip()) + if value and value[0] in "|>": + commands = [b for b in body if b and not b.startswith("#")] + commands = [_strip_shell_comment(b).strip() for b in commands] + commands = [b for b in commands if b] + else: + commands = [] + else: + commands = [re.sub(r"\s+#.*$", "", value).strip("'\"")] + commands = [_strip_shell_comment(c).strip() for c in commands] + commands = [c for c in commands if c] + found.append((number, commands)) + return found + + +def noop_steps(text): + found = [] + for number, commands in _workflow_commands(text): + if not commands: + found.append({"line": number, "run": ""}) + if commands and all(c in NOOP_RUN for c in commands): + found.append({"line": number, "run": " ; ".join(commands)}) + return found + + +def tool_paths(files): + return sorted(p for p in files if p.startswith("tools/") and p.endswith(CODE_SUFFIXES) + and area(p) == "tooling" and not p.endswith("__init__.py")) + + +def _module(path): + return path.rsplit("/", 1)[-1].rsplit(".", 1)[0] + + +def _names_tool(text, path, unique): + # The full path always counts. The module name and the basename count only + # when no other tracked file shares the basename, so neither is ambiguous. + pattern = r"\b" + re.escape(path) + r"\b" + if unique: + module = re.escape(_module(path)) + pattern += (r"|\b(?:import|from)\s+(?:[\w.]+\.)?" + module + r"\b" + r"|\bfrom\s+[\w.]+\s+import\s+(?:\([^)]*|[^\n]*)\b" + module + r"\b" + r"|\b" + re.escape(path.rsplit("/", 1)[-1]) + r"\b") + return re.search(pattern, text) is not None + + + +def _tool_for_module(module, tools, unique): + if not module: + return None + module = module.strip(".") + candidate = module.replace(".", "/") + ".py" + if candidate in tools: + return candidate + basename = module.rsplit(".", 1)[-1] + if unique.get(basename): + for path in tools: + if path.rsplit("/", 1)[-1] == basename + ".py": + return path + return None + + +def _tool_for_token(token, tools): + token = token.strip().strip("'\"").replace("\\", "/") + while token.startswith("./"): + token = token[2:] + if token in tools: + return token + for path in tools: + if token.endswith("/" + path): + return path + return None + + +def python_tool_references(text, tools, unique): + references = set() + try: + tree = ast.parse(text) + except (SyntaxError, ValueError): + return references + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + path = _tool_for_module(alias.name, tools, unique) + if path: + references.add(path) + elif isinstance(node, ast.ImportFrom) and node.level == 0: + module = node.module or "" + path = _tool_for_module(module, tools, unique) + if path: + references.add(path) + else: + for alias in node.names: + path = _tool_for_module((module + "." + alias.name).strip("."), tools, unique) + if path: + references.add(path) + elif isinstance(node, ast.Call): + call_name = _dotted_name(node.func) + if call_name in {"subprocess.run", "subprocess.call", "subprocess.Popen", + "subprocess.check_call", "subprocess.check_output", "os.system", + "runpy.run_path"}: + for arg in node.args: + if isinstance(arg, (ast.List, ast.Tuple)): + tokens = [elt.value for elt in arg.elts + if isinstance(elt, ast.Constant) and isinstance(elt.value, str)] + elif isinstance(arg, ast.Constant) and isinstance(arg.value, str): + try: + tokens = shlex.split(arg.value) + except ValueError: + tokens = [] + else: + tokens = [] + if tokens and tokens[0] in {"python", "python3", "python3.11", "bash", "sh"}: + for token in tokens[1:]: + if token == "-m": + continue + path = _tool_for_token(token, tools) + if path: + references.add(path) + break + elif tokens: + path = _tool_for_token(tokens[0], tools) + if path: + references.add(path) + return references + + +def _strip_js_comments(text): + out = [] + i = 0 + quote = None + line_comment = False + block_comment = False + while i < len(text): + ch = text[i] + nxt = text[i + 1] if i + 1 < len(text) else "" + if line_comment: + if ch == "\n": + line_comment = False + out.append(ch) + else: + out.append(" ") + i += 1 + continue + if block_comment: + if ch == "*" and nxt == "/": + out.extend((" ", " ")) + block_comment = False + i += 2 + else: + out.append("\n" if ch == "\n" else " ") + i += 1 + continue + if quote: + out.append(ch) + if ch == "\\" and i + 1 < len(text): + out.append(text[i + 1]) + i += 2 + continue + if ch == quote: + quote = None + i += 1 + continue + if ch in "\"'" + chr(96): + quote = ch + out.append(ch) + i += 1 + continue + if ch == "/" and nxt == "/": + line_comment = True + out.extend((" ", " ")) + i += 2 + continue + if ch == "/" and nxt == "*": + block_comment = True + out.extend((" ", " ")) + i += 2 + continue + out.append(ch) + i += 1 + return "".join(out) + + +def js_tool_references(text, source_path, tools): + references = set() + visible = _strip_js_comments(text) + pattern = re.compile(r"(?m)^\s*import\s+(?:[^'\";]+?\s+from\s+)?['\"]([^'\"]+)['\"]\s*;?") + for match in pattern.finditer(visible): + specifier = match.group(1) + if specifier.startswith("."): + target = posixpath.normpath(posixpath.join(posixpath.dirname(source_path), specifier)) + else: + target = specifier + if target in tools: + references.add(target) + return references + + +def workflow_tool_references(text, tools): + references = set() + for _number, commands in _workflow_commands(text): + for command in commands: + try: + tokens = shlex.split(command) + except ValueError: + continue + if not tokens: + continue + if tokens[0] in {"python", "python3", "python3.11", "bash", "sh"}: + for index, token in enumerate(tokens[1:], 1): + if token == "-m" and index + 1 < len(tokens): + module_path = tokens[index + 1].replace(".", "/") + ".py" + if module_path in tools: + references.add(module_path) + break + if token.startswith("-"): + continue + path = _tool_for_token(token, tools) + if path: + references.add(path) + break + else: + path = _tool_for_token(tokens[0], tools) + if path: + references.add(path) + return references + + +def _markdown_visible(text): + return re.sub(r"", "", text, flags=re.S) + + +def tool_report(files, texts, all_paths): + tools = tool_paths(files) + basenames = {} + for p in all_paths: + name = p.rsplit("/", 1)[-1] + basenames[name] = basenames.get(name, 0) + 1 + unique = {p: basenames[p.rsplit("/", 1)[-1]] == 1 for p in tools} + unique_modules = {p.rsplit("/", 1)[-1].rsplit(".", 1)[0]: unique[p] for p in tools} + docs = [p for p in texts if p.endswith(".md")] + exercisers = [p for p in texts if area(p) == "tests" or + (p.startswith(".github/workflows/") and p.endswith((".yml", ".yaml")))] + direct, tool_dependencies = {p: [] for p in tools}, {} + for path in tools: + tool_dependencies[path] = python_tool_references(texts.get(path, ""), tools, unique_modules) + for exerciser in exercisers: + if exerciser.endswith(".py"): + refs = python_tool_references(texts[exerciser], tools, unique_modules) + elif exerciser.endswith((".mjs", ".js", ".cjs")): + refs = js_tool_references(texts[exerciser], exerciser, tools) + else: + refs = workflow_tool_references(texts[exerciser], tools) + for path in refs: + direct[path].append(exerciser) + report = {} + for path in tools: + documented = [d for d in docs if _names_tool(_markdown_visible(texts[d]), path, unique[path])] + direct_paths = direct[path] + docstring = None + if path.endswith(".py") and path in texts: + try: + docstring = bool(ast.get_docstring(ast.parse(files[path]))) + except (SyntaxError, ValueError): + docstring = False + report[path] = {"documented_by": documented[:5], "documented": bool(documented), + "exercised_by": direct_paths[:5], "exercised": bool(direct_paths), + "module_docstring": docstring} + changed = True + while changed: + changed = False + for path in tools: + if report[path]["exercised"] or path not in texts: + continue + for other in tools: + if other != path and report[other]["exercised"] and other in texts and path in tool_dependencies.get(other, set()): + report[path]["exercised"] = True + report[path]["exercised_by"] = ["via " + other] + changed = True + break + return report + + +def census(files, all_paths=None): + """Census of a {path: bytes} tree. Pure: no I/O. `all_paths` lists every + tracked path (default: the keys of `files`) for basename ambiguity.""" + findings, texts = [], {} + for path in sorted(files): + raw = files[path] + kind = area(path) + if not path.endswith(SCANNED): + continue + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + findings.append({"path": path, "area": kind, "class": "undecodable", "line": 0, "detail": "not UTF-8"}) + continue + texts[path] = text + if path.endswith(".py"): + try: + tree = ast.parse(raw, filename=path) + markers, mentions = python_words(raw) + except (SyntaxError, ValueError, tokenize.TokenError) as exc: + findings.append({"path": path, "area": kind, "class": "unparseable", "line": 0, "detail": str(exc)[:120]}) + continue + imports = _import_bindings(tree) + for stub in python_stubs(tree, imports): + findings.append({"path": path, "area": kind, "class": "stub" if not stub["context"] else "stub-context", + "line": stub["line"], "detail": "%s: %s%s" % (stub["name"], stub["kind"], + "; " + stub["context"] if stub["context"] else "")}) + elif path.endswith((".mjs", ".js", ".cjs", ".sh")): + markers, mentions = text_words(text, path) + elif path.endswith((".yml", ".yaml")): + markers, mentions = yaml_words(text) + else: + markers, mentions = [], [] + findings.extend({"path": path, "area": kind, "class": "marker", "line": m["line"], "detail": m["word"]} for m in markers) + findings.extend({"path": path, "area": kind, "class": "mention", "line": m["line"], "detail": m["word"]} for m in mentions) + if path.startswith(".github/workflows/") and path.endswith((".yml", ".yaml")): + findings.extend({"path": path, "area": kind, "class": "noop", "line": n["line"], "detail": n["run"]} + for n in noop_steps(text)) + tools = tool_report(files, texts, sorted(all_paths if all_paths is not None else files)) + failing = [f for f in findings if f["area"] in PRODUCTION + and f["class"] in {"stub", "marker", "noop", "undecodable", "unparseable"}] + failing_tools = sorted(p for p, t in tools.items() if not (t["documented"] and t["exercised"])) + return {"findings": findings, "tools": tools, "failing": failing, "failing_tools": failing_tools, + "status": "FAIL" if failing or failing_tools else "PASS"} + + +def read_tree(root, rev): + listing = subprocess.run(["git", "-C", root, "ls-tree", "-r", "-z", "--full-tree", rev], + check=True, capture_output=True).stdout + entries, skipped, tracked = [], [], [] + for item in listing.split(b"\0"): + if not item: + continue + meta, path = item.split(b"\t", 1) + mode, kind, sha = meta.split() + path = path.decode("utf-8", "surrogateescape") + if kind == b"blob" and mode != b"120000": + if path.endswith(SCANNED): + entries.append((path, sha.decode())) + tracked.append(path) + else: + skipped.append({"path": path, "mode": mode.decode(), "type": kind.decode()}) + batch = subprocess.run(["git", "-C", root, "cat-file", "--batch"], check=True, capture_output=True, + input=b"".join(sha.encode() + b"\n" for _p, sha in entries)).stdout + files, at = {}, 0 + for path, sha in entries: + end = batch.index(b"\n", at) + header = batch[at:end].split() + if len(header) != 3 or header[0].decode() != sha: + raise ValueError("unexpected git cat-file header for " + path) + size = int(header[2]) + files[path] = batch[end + 1:end + 1 + size] + at = end + 1 + size + 1 + return files, skipped, tracked + + +def main(argv=None): + parser = argparse.ArgumentParser(description="CBZ-0 G11 census: stub scan and tool documentation check.") + parser.add_argument("--root", default=".", help="git repository (default: current directory)") + parser.add_argument("--rev", default="HEAD", help="revision to read (default: HEAD)") + parser.add_argument("--json", help="write the full report to this file") + args = parser.parse_args(argv) + try: + commit = subprocess.run(["git", "-C", args.root, "rev-parse", "--verify", args.rev + "^{commit}"], + check=True, capture_output=True, text=True).stdout.strip() + files, skipped, tracked = read_tree(args.root, commit) + except (subprocess.CalledProcessError, OSError, ValueError) as exc: + print("G11_CENSUS_ERROR: cannot read revision %s: %s" % (args.rev, exc), file=sys.stderr) + return 2 + report = census(files, tracked) + report.update(commit=commit, files=len(files), tracked=len(tracked), skipped=skipped) + if args.json: + with open(args.json, "w", encoding="utf-8") as fh: + json.dump(report, fh, indent=1, sort_keys=True) + fh.write("\n") + for f in report["failing"]: + print("%s:%d: %s %s (%s)" % (f["path"], f["line"], f["area"], f["class"], f["detail"])) + for path in report["failing_tools"]: + t = report["tools"][path] + print("%s: tool %s" % (path, ", ".join(w for w, ok in (("undocumented", t["documented"]), + ("unexercised", t["exercised"])) if not ok))) + counts = {} + for f in report["findings"]: + counts[f["class"]] = counts.get(f["class"], 0) + 1 + print("G11_CENSUS_%s: commit %s, %d files, %d tools, findings %s, failing %d, failing tools %d" + % (report["status"], commit, len(files), len(report["tools"]), json.dumps(counts, sort_keys=True), + len(report["failing"]), len(report["failing_tools"]))) + return 1 if report["status"] == "FAIL" else 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file From d566f1591fb694aeb0a8d54ede34b5f9285519cd Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:31:03 -0500 Subject: [PATCH 02/27] CBZ-0 G11: add adversarial auditor controls --- tests/test_g11_census.py | 340 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 340 insertions(+) create mode 100644 tests/test_g11_census.py diff --git a/tests/test_g11_census.py b/tests/test_g11_census.py new file mode 100644 index 000000000..54ecc91b7 --- /dev/null +++ b/tests/test_g11_census.py @@ -0,0 +1,340 @@ +"""Tests for the CBZ-0 G11 census (tools/cbz/g11_census.py). + +Every finding class is planted once in a synthetic tree, beside look-alikes that +must not be flagged; the mutation self-test (tools/cbz/g11_mutation_selftest.py) +requires these tests to fail when any detector is weakened. +""" +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools" / "cbz")) +import g11_census as g11 # noqa: E402 +import g11_mutation_selftest as g11m # noqa: E402 + +DOCUMENTED_TESTED = { + "tools/README.md": b"Run `tools/good.py` to check things.\n", + "tests/test_good.py": b"import good\n", +} + + +def run(tree, all_paths=None): + return g11.census({k: (v.encode() if isinstance(v, str) else v) for k, v in tree.items()}, all_paths) + + +def classes(report, path=None, area=None): + return sorted((f["class"], f["line"]) for f in report["findings"] + if (path is None or f["path"] == path) and (area is None or f["area"] == area)) + + +def stubs(text, path="tools/good.py"): + return [(f["line"], f["detail"]) for f in run({path: text})["findings"] + if f["class"] in ("stub", "stub-context")] + + +class Areas(unittest.TestCase): + def test_area_of_each_path_family(self): + cases = {"tools/a.py": "tooling", ".github/workflows/x.yml": "tooling", + "docs/site/app.mjs": "site", "docs/site/vendor/three.js": "vendor", + "tests/test_a.py": "tests", "tools/x/test_y.py": "tests", "conftest.py": "tests", + "docs/site/app.test.mjs": "tests", "experiments/e.py": "research", + "reviews/r/s.py": "evidence", "docs/reports/h.py": "evidence"} + for path, expected in cases.items(): + with self.subTest(path=path): + self.assertEqual(g11.area(path), expected) + + +class PythonStubs(unittest.TestCase): + def test_each_stub_body(self): + self.assertEqual(stubs("def a():\n pass\n"), [(1, "a: pass")]) + self.assertEqual(stubs("def a():\n ...\n"), [(1, "a: ellipsis")]) + self.assertEqual(stubs("def a():\n raise NotImplementedError\n"), [(1, "a: raise NotImplementedError")]) + self.assertEqual(stubs("def a():\n raise NotImplementedError('later')\n"), + [(1, "a: raise NotImplementedError")]) + self.assertEqual(stubs('def a():\n """Only a docstring."""\n'), [(1, "a: docstring only")]) + + def test_docstring_then_stub_body(self): + self.assertEqual(stubs('def a():\n """Doc."""\n pass\n'), [(1, "a: pass")]) + + def test_methods_async_and_nested(self): + text = ("class K:\n def m(self):\n pass\n" + "async def b():\n ...\n" + "def outer():\n def inner():\n pass\n return inner\n") + self.assertEqual(stubs(text), [(2, "m: pass"), (4, "b: ellipsis"), (7, "inner: pass")]) + + def test_real_bodies_are_not_stubs(self): + for text in ("def a():\n return 1\n", + 'def a():\n """Doc."""\n return 1\n', + "def a(x):\n if x:\n pass\n return x\n", + "def a():\n pass\n return 2\n", + "def a():\n raise ValueError('bad')\n", + "class E(Exception):\n pass\n"): + with self.subTest(text=text): + self.assertEqual(stubs(text), []) + + def test_legitimate_contexts_are_reported_but_do_not_fail(self): + text = ("from abc import ABC, abstractmethod\nfrom typing import Protocol, overload\n" + "class Base(ABC):\n def m(self):\n ...\n" + "class Shape(Protocol):\n def area(self) -> float:\n ...\n" + "class Bad(ValueError):\n def __init__(self):\n pass\n" + "class C:\n @abstractmethod\n def a(self):\n pass\n" + " @overload\n def f(self, x: int) -> int:\n ...\n") + report = run({"tools/good.py": text, **DOCUMENTED_TESTED}) + contexts = sorted(f["detail"] for f in report["findings"] if f["class"] == "stub-context") + self.assertEqual(contexts, ["__init__: pass; method of an exception class", + "a: pass; @abstractmethod", "area: ellipsis; method of a Protocol class", + "f: ellipsis; @overload", "m: ellipsis; method of a ABC class"]) + self.assertEqual(report["failing"], []) + self.assertEqual(report["status"], "PASS") + + def test_request_handler_log_message_override_is_legitimate(self): + text = ("from http.server import SimpleHTTPRequestHandler\n" + "class Handler(SimpleHTTPRequestHandler):\n def log_message(self, *args):\n pass\n" + " def do_GET(self):\n pass\n" + "class Other:\n def log_message(self, *args):\n pass\n") + found = sorted((f["class"], f["line"]) for f in run({"tools/good.py": text})["findings"] + if f["class"].startswith("stub")) + self.assertEqual(found, [("stub", 5), ("stub", 8), ("stub-context", 3)]) + + def test_production_stub_fails_but_research_stub_does_not(self): + bad = run({"tools/good.py": "def a():\n pass\n", **DOCUMENTED_TESTED}) + self.assertEqual([(f["path"], f["class"]) for f in bad["failing"]], [("tools/good.py", "stub")]) + self.assertEqual(bad["status"], "FAIL") + research = run({"experiments/e.py": "def a():\n pass\n"}) + self.assertEqual(classes(research), [("stub", 1)]) + self.assertEqual(research["status"], "PASS") + + +class Markers(unittest.TestCase): + def test_python_comment_is_marker_and_string_is_mention(self): + text = ('def a():\n # TODO: finish\n return "TODO in data"\n' + '# FIXME later\nx = 1 # XXX\n# TBD\n# STUB\n# not implemented yet\n') + self.assertEqual(classes(run({"tools/good.py": text}), "tools/good.py"), + [("marker", 2), ("marker", 4), ("marker", 5), ("marker", 6), ("marker", 7), + ("marker", 8), ("mention", 3)]) + + def test_lowercase_words_and_lookalikes_are_not_markers(self): + text = "# todo list, xxx-large, stubs, tbd-free\nTODOS = 1 # TODOS are fine\n" + self.assertEqual(classes(run({"tools/good.py": text}), "tools/good.py"), []) + + def test_docstring_marker_is_a_mention(self): + self.assertEqual(classes(run({"tools/good.py": 'def a():\n """TODO: doc."""\n return 1\n'})), + [("mention", 2)]) + + def test_javascript_comments_and_strings(self): + text = ('// TODO top\nconst s = "TODO not a comment";\nconst u = "http://x"; // FIXME tail\n' + "/* XXX\n still TBD */ const t = 'STUB';\nconst r = `// TODO in template`;\n") + self.assertEqual(classes(run({"docs/site/app.mjs": text})), + [("marker", 1), ("marker", 3), ("marker", 4), ("marker", 5), + ("mention", 2), ("mention", 5), ("mention", 6)]) + + def test_escaped_quote_inside_string(self): + text = 'const s = "a \\" // TODO still data";\n' + self.assertEqual(classes(run({"docs/site/app.mjs": text})), [("mention", 1)]) + + def test_shell_comments_and_strings(self): + text = '#!/bin/sh\n# TODO setup\necho "TODO in data" # FIXME\nn=${#TODO}\n' + self.assertEqual(classes(run({"tools/good.sh": text})), + [("marker", 2), ("marker", 3), ("mention", 3), ("mention", 4)]) + + def test_production_marker_fails(self): + report = run({"tools/good.py": "# TODO\n", **DOCUMENTED_TESTED}) + self.assertEqual(report["status"], "FAIL") + + def test_site_marker_fails_and_vendor_marker_does_not(self): + self.assertEqual(run({"docs/site/app.mjs": "// FIXME\n"})["status"], "FAIL") + self.assertEqual(run({"docs/site/vendor/lib.js": "// FIXME\n"})["status"], "PASS") + + +class Workflows(unittest.TestCase): + def test_noop_steps(self): + text = ("jobs:\n a:\n steps:\n" + " - run: true\n" + " - name: colon\n run: ':'\n" + " - run: exit 0 # placeholder\n" + " - name: block\n run: |\n # nothing\n true\n shell: bash\n" + " - run: |\n true\n shell: bash\n" + " - run: echo hi\n" + " - name: real block\n run: |\n true\n make test\n") + found = [f["line"] for f in run({".github/workflows/w.yml": text})["findings"] if f["class"] == "noop"] + self.assertEqual(found, [4, 6, 7, 9, 13]) + + def test_production_noop_fails(self): + self.assertEqual(run({".github/workflows/w.yml": " - run: true\n"})["status"], "FAIL") + self.assertEqual(run({".github/workflows/w.yml": " - run: make\n"})["status"], "PASS") + + +class ToolChecks(unittest.TestCase): + def tools(self, tree, all_paths=None): + return run(tree, all_paths)["tools"] + + def test_documented_and_exercised_by_path_import_and_basename(self): + tree = {"tools/a.py": "x = 1\n", "tools/b.py": "x = 1\n", "tools/c.mjs": "export const x = 1;\n", "tools/d.py": "x = 1\n", + "docs/USE.md": "See tools/a.py, b.py, c.mjs and d.py.\n", + "tests/test_a.py": "from tools import helper_x, a\n", + "tests/test_d.py": "import d\n", + ".github/workflows/ci.yml": " - run: python tools/b.py\n", + "tests/c.test.mjs": "import { x } from '../tools/c.mjs';\n"} + report = self.tools(tree) + for path in ("tools/a.py", "tools/b.py", "tools/c.mjs", "tools/d.py"): + with self.subTest(tool=path): + self.assertTrue(report[path]["documented"]) + self.assertTrue(report[path]["exercised"]) + + def test_undocumented_and_unexercised_tools_fail(self): + result = run({"tools/lonely.py": "x = 1\n", "tools/README.md": "nothing here\n"}) + self.assertEqual(result["failing_tools"], ["tools/lonely.py"]) + self.assertFalse(result["tools"]["tools/lonely.py"]["documented"]) + self.assertFalse(result["tools"]["tools/lonely.py"]["exercised"]) + self.assertEqual(result["status"], "FAIL") + + def test_exercised_through_an_exercised_tool(self): + tree = {"tools/top.py": "import helper\n", "tools/helper.py": "x = 1\n", "tools/orphan.py": "x = 1\n", + "tools/README.md": "tools/top.py tools/helper.py tools/orphan.py\n", + "tests/test_top.py": "import top\n"} + report = self.tools(tree) + self.assertEqual(report["tools/helper.py"]["exercised_by"], ["via tools/top.py"]) + self.assertFalse(report["tools/orphan.py"]["exercised"]) + + def test_ambiguous_basename_needs_the_full_path(self): + tree = {"tools/run.py": "x = 1\n", "docs/USE.md": "call run.py\n", "tests/test_x.py": "import run\n"} + ambiguous = self.tools(tree, sorted(tree) + ["experiments/run.py"]) + self.assertFalse(ambiguous["tools/run.py"]["documented"]) + self.assertFalse(ambiguous["tools/run.py"]["exercised"]) + unique = self.tools(tree) + self.assertTrue(unique["tools/run.py"]["documented"]) + self.assertTrue(unique["tools/run.py"]["exercised"]) + + def test_full_path_counts_even_when_basename_is_ambiguous(self): + tree = {"tools/run.py": "x = 1\n", "docs/USE.md": "call tools/run.py\n", + ".github/workflows/ci.yml": " - run: python tools/run.py\n"} + report = self.tools(tree, sorted(tree) + ["experiments/run.py"]) + self.assertTrue(report["tools/run.py"]["documented"]) + self.assertTrue(report["tools/run.py"]["exercised"]) + + def test_module_docstring_is_recorded(self): + report = self.tools({"tools/a.py": '"""Does a."""\n', "tools/b.py": "x = 1\n"}) + self.assertEqual((report["tools/a.py"]["module_docstring"], report["tools/b.py"]["module_docstring"]), + (True, False)) + + def test_tests_and_init_files_are_not_tools(self): + report = self.tools({"tools/__init__.py": "", "tools/x/test_y.py": "x = 1\n", "tools/a.py": "x = 1\n"}) + self.assertEqual(sorted(report), ["tools/a.py"]) + + +class FailClosed(unittest.TestCase): + def test_undecodable_and_unparseable_production_files_fail(self): + for raw, expected in ((b"x = '\xff'\n", "undecodable"), (b"def broken(:\n", "unparseable")): + with self.subTest(raw=raw): + report = run({"tools/good.py": raw, **DOCUMENTED_TESTED}) + self.assertEqual(report["status"], "FAIL") + self.assertEqual([f["class"] for f in report["failing"]], [expected]) + + def test_unparseable_research_file_is_reported_not_failed(self): + report = run({"experiments/old.py": b"print 'python 2'\n"}) + self.assertEqual(classes(report), [("unparseable", 0)]) + self.assertEqual(report["status"], "PASS") + + +class CommandLine(unittest.TestCase): + def repo(self, files): + d = tempfile.mkdtemp() + self.addCleanup(lambda: subprocess.run(["rm", "-rf", d])) + env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@invalid", + GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="t@invalid") + subprocess.run(["git", "init", "-q", d], check=True) + for path, text in files.items(): + Path(d, path).parent.mkdir(parents=True, exist_ok=True) + Path(d, path).write_bytes(text) + subprocess.run(["git", "-C", d, "add", "-A"], check=True) + subprocess.run(["git", "-C", d, "commit", "-q", "-m", "fixture"], check=True, env=env) + return d + + def cli(self, root, *args): + script = str(ROOT / "tools" / "cbz" / "g11_census.py") + return subprocess.run([sys.executable, "-B", "-S", script, "--root", root, *args], + capture_output=True, text=True) + + def test_exit_codes_and_exact_committed_bytes(self): + good = {"tools/good.py": b"x = 1\n", "tools/README.md": b"tools/good.py\n", "tests/test_good.py": b"import good\n"} + root = self.repo(good) + self.assertEqual(self.cli(root).returncode, 0) + Path(root, "tools/good.py").write_bytes(b"def a():\n pass\n") # uncommitted: not read + self.assertEqual(self.cli(root).returncode, 0) + bad = self.repo({**good, "tools/good.py": b"def a():\n pass\n"}) + result = self.cli(bad) + self.assertEqual(result.returncode, 1) + self.assertIn("tools/good.py:1: tooling stub (a: pass)", result.stdout) + self.assertEqual(self.cli(root, "--rev", "no-such-revision").returncode, 2) + + + + +class AdversarialControls(unittest.TestCase): + def test_decoy_names_do_not_exempt_stubs(self): + text = ( + "from abc import abstractmethod as am\nimport typing as t\n" + "def abstractmethod(fn): return fn\n" + "class Fake:\n @staticmethod\n def abstractmethod(fn): return fn\n" + "class Protocol: pass\nclass ABC: pass\nclass ValueError: pass\n" + "@abstractmethod\ndef decoy_decorator():\n pass\n" + "@Fake.abstractmethod\ndef decoy_qualified():\n pass\n" + "class NotReallyError:\n def fail(self):\n pass\n" + "class NotReallyProtocol:\n def fail(self):\n ...\n" + "class NotReallyABC:\n def fail(self):\n pass\n" + "class NotReallyRequestHandler:\n def log_message(self, *args):\n pass\n" + "class DecoyProtocol(Protocol):\n def proto(self):\n pass\n" + "class DecoyABC(ABC):\n def abc(self):\n pass\n" + "class ShadowedBuiltin(ValueError):\n def builtin(self):\n pass\n" + "class Genuine:\n @am\n def abstract(self):\n pass\n" + "class Shape(t.Protocol):\n def area(self):\n ...\n" + ) + report = run({"tools/good.py": text, **DOCUMENTED_TESTED}) + self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]), + ["abc", "builtin", "decoy_decorator", "decoy_qualified", "fail", "fail", "fail", "log_message", "proto"]) + self.assertEqual(report["status"], "FAIL") + + def test_comment_and_string_mentions_do_not_exercise_tools(self): + tree = { + "tools/runner.py": "def main():\n return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + "tests/test_runner.py": "# tools/runner.py\nDOC = 'tools/runner.py'\nprint('tools/runner.py')\n", + } + report = run(tree) + self.assertFalse(report["tools/runner.py"]["exercised"]) + self.assertEqual(report["failing_tools"], ["tools/runner.py"]) + + def test_empty_and_comment_only_run_blocks_fail(self): + text = ("jobs:\n a:\n steps:\n" + " - run: |\n # only comment\n" + " - run: ''\n" + " - run: # empty\n" + " - run: true\n") + found = [f["line"] for f in run({".github/workflows/w.yml": text})["findings"] + if f["class"] == "noop"] + self.assertEqual(found, [4, 6, 7, 8]) + + def test_yaml_comments_are_scanned_for_markers(self): + text = ("name: ci\n# TODO workflow comment\njobs:\n a:\n steps:\n" + " - run: echo '# FIXME in data'\n") + report = run({".github/workflows/w.yml": text}) + self.assertEqual(classes(report), [("marker", 2), ("mention", 6)]) + + +class MutationAttribution(unittest.TestCase): + def test_mutation_kill_requires_exact_target_assertion(self): + expected = "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion" + good = {"exit": 1, "ran": 1, "failures": 1, "errors": 0, "skipped": 0, + "test_id": expected, "traceback_method": "test_mutation_kill_requires_exact_target_assertion", + "traceback_line": 17, "exception_type": "AssertionError"} + self.assertTrue(g11m.qualifies_kill(good, expected, 17)) + wrong_line = dict(good, traceback_line=18) + self.assertFalse(g11m.qualifies_kill(wrong_line, expected, 17)) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From fe8a330f41eb905f740d33bafed25e209e9e7a84 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:31:06 -0500 Subject: [PATCH 03/27] CBZ-0 G11: require designated assertion mutation kills --- tools/cbz/g11_mutation_selftest.py | 366 +++++++++++++++++++++++++++++ 1 file changed, 366 insertions(+) create mode 100644 tools/cbz/g11_mutation_selftest.py diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py new file mode 100644 index 000000000..828864100 --- /dev/null +++ b/tools/cbz/g11_mutation_selftest.py @@ -0,0 +1,366 @@ +#!/usr/bin/env python3 +"""Mutation self-test of the G11 census (tools/cbz/g11_census.py). + +Each mutation weakens the census in one way. For every mutation the harness +copies the census and its test module into an isolated directory, applies the +mutation, and runs the test module there. A mutation is KILLED only when the +test run reports at least one assertion failure and no error, so a mutant that +breaks the import or crashes the tests does not count. The self-test passes +only when the unmutated copy passes and every mutation is killed. + +A mutation whose target text does not occur exactly once is a harness error, +never a silent no-op. + +Usage: python3 -B -S tools/cbz/g11_mutation_selftest.py [--root DIR] [--json] +Exit status: 0 pass, 1 a mutation survived or the baseline failed, 2 harness error. +""" +from __future__ import annotations + +import argparse +import ast +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +TOOL = os.path.join("tools", "cbz", "g11_census.py") +TESTS = os.path.join("tests", "test_g11_census.py") +HARNESS = os.path.join("tools", "cbz", "g11_mutation_selftest.py") + +MUTATIONS = [ + ("pass not detected", ' if isinstance(stmt, ast.Pass):\n return "pass"\n', + ' if False:\n return "pass"\n'), + ("ellipsis not detected", "and stmt.value.value is Ellipsis:", "and stmt.value.value is None:"), + ("NotImplementedError not detected", 'exc.id == "NotImplementedError"', 'exc.id == "NotImplemented"'), + ("raise call not unwrapped", "exc = stmt.exc.func if isinstance(stmt.exc, ast.Call) else stmt.exc", + "exc = stmt.exc"), + ("docstring-only not reported", ' if not rest:\n return "docstring only"\n', + " if not rest:\n return None\n"), + ("docstring not stripped", " rest = rest[1:]\n", " rest = rest[0:]\n"), + ("decorator context ignored", " if _trusted_decorator(dec, imports):", " if False:"), + ("Protocol/ABC context ignored", " if _trusted_abstract_base(base, imports):", " if False:"), + ("exception context ignored", " if _builtin_exception_base(base, imports):", " if False:"), + ("log_message context ignored", ' if _http_request_handler_base(base, imports) and func.name == "log_message":', " if False:"), + ("log_message context too broad", ' if _http_request_handler_base(base, imports) and func.name == "log_message":', + ' if _http_request_handler_base(base, imports):'), + ("module docstring not recorded", " docstring = bool(ast.get_docstring(ast.parse(files[path])))", + " docstring = True"), + ("class bodies not visited", " visit(child, classes + [child])", " pass"), + ("nested functions not visited", " visit(child, [])\n else:", + " pass\n else:"), + ("comments read as data", " if tok.type == tokenize.COMMENT:\n target = markers", + " if tok.type == tokenize.COMMENT:\n target = mentions"), + ("strings read as comments", " target = mentions\n else:\n continue", + " target = markers\n else:\n continue"), + ("marker words case-insensitive", r'MARKER = re.compile(r"\b(?:TODO|FIXME|XXX|TBD|STUB)\b|\b(?i:not implemented)\b")', + r'MARKER = re.compile(r"(?i)\b(?:TODO|FIXME|XXX|TBD|STUB)\b|\bnot implemented\b")'), + ("marker word boundary dropped", r'MARKER = re.compile(r"\b(?:TODO|FIXME|XXX|TBD|STUB)\b|\b(?i:not implemented)\b")', + r'MARKER = re.compile(r"(?:TODO|FIXME|XXX|TBD|STUB)|\b(?i:not implemented)\b")'), + ("quotes not tracked", " elif ch in \"\\\"'`\":\n quote = ch", + " elif False:\n quote = ch"), + ("string escapes ignored", ' if ch == "\\\\":\n i += 2\n', + " if False:\n i += 2\n"), + ("hash comment needs no whitespace", '(opener != "#" or i == 0 or line[i - 1].isspace())', "True"), + ("block comments not entered", " in_block, rest = True, rest[first + 2:]", + " in_block, rest = False, rest[first + 2:]"), + ("block comment end ignored", " in_block, rest = False, rest[end + 2:]", + " in_block, rest = True, rest[end + 2:]"), + ("mentions dropped", "for m in mentions)", "for m in [])"), + ("noop detection off", " if commands and all(c in NOOP_RUN for c in commands):", " if False:"), + ("block scalar not read", ' if not value or value[0] in "|>":', " if False:"), + ("block scalar ends late", " if nxt.strip() and len(nxt) - len(nxt.lstrip()) <= key_column:", + " if nxt.strip() and len(nxt) - len(nxt.lstrip()) < key_column:"), + ("inline comment kept", ' commands = [re.sub(r"\\s+#.*$", "", value).strip("\'\\"")]', + ' commands = [value.strip("\'\\"")]'), + ("tests counted as tools", ' and area(p) == "tooling" and not p.endswith("__init__.py"))', + ' and True and not p.endswith("__init__.py"))'), + ("__init__ counted as tool", ' and area(p) == "tooling" and not p.endswith("__init__.py"))', + ' and area(p) == "tooling" and True)'), + ("basename ambiguity ignored", ' unique = {p: basenames[p.rsplit("/", 1)[-1]] == 1 for p in tools}', + " unique = {p: True for p in tools}"), + ("path reference not recognised", ' pattern = r"\\b" + re.escape(path) + r"\\b"\n', + ' pattern = r"\\bNOMATCH\\b"\n'), + ("import not recognised", " if isinstance(node, ast.Import):", " if False:"), + ("from-import list not recognised", " elif isinstance(node, ast.ImportFrom) and node.level == 0:", " elif False:"), + ("basename not recognised", ' r"|\\b" + re.escape(path.rsplit("/", 1)[-1]) + r"\\b")', + ' r"|\\bNOMATCH")'), + ("transitive exercise off", ' if other != path and report[other]["exercised"] and other in texts and path in tool_dependencies.get(other, set()):', + ' if False and other != path and report[other]["exercised"] and other in texts and path in tool_dependencies.get(other, set()):'), + ("site not production", 'PRODUCTION = {"tooling", "site"}', 'PRODUCTION = {"tooling"}'), + ("research counted as production", 'PRODUCTION = {"tooling", "site"}', 'PRODUCTION = {"tooling", "site", "research"}'), + ("vendor counted as site", ' if path.startswith("docs/site/vendor/"):\n return "vendor"', + ' if False:\n return "vendor"'), + ("undecodable decoded leniently", ' text = raw.decode("utf-8")', ' text = raw.decode("utf-8", "replace")'), + ("unparseable not a failure class", '"class": "unparseable"', '"class": "mention"'), + ("legitimate context still fails", '"class": "stub" if not stub["context"] else "stub-context"', '"class": "stub"'), + ("markers do not fail", 'f["class"] in {"stub", "marker", "noop", "undecodable", "unparseable"}]', + 'f["class"] in {"stub", "noop", "undecodable", "unparseable"}]'), + ("noop steps do not fail", 'f["class"] in {"stub", "marker", "noop", "undecodable", "unparseable"}]', + 'f["class"] in {"stub", "marker", "undecodable", "unparseable"}]'), + ("tool findings do not fail", '"status": "FAIL" if failing or failing_tools else "PASS"}', + '"status": "FAIL" if failing else "PASS"}'), + ("exit status always zero", ' return 1 if report["status"] == "FAIL" else 0\n', " return 0\n"), + ("unreadable revision exits zero", + ' print("G11_CENSUS_ERROR: cannot read revision %s: %s" % (args.rev, exc), file=sys.stderr)\n return 2\n', + ' print("G11_CENSUS_ERROR: cannot read revision %s: %s" % (args.rev, exc), file=sys.stderr)\n return 0\n'), +] + +RESULT = re.compile(r"^(FAILED|OK)\b(?: \(([^)]*)\))?\s*$", re.M) +SUMMARY = re.compile(r"Ran (\d+) tests? in [^\n]+") +FAIL_HEADER = re.compile(r"^FAIL: ([^(]+) \(([^)]+)\)$", re.M) +TRACE_FRAME = re.compile(r'File "([^"]*test_g11_census\.py)", line (\d+), in ([A-Za-z_]\w*)') +EXCEPTION = re.compile(r"(?m)^(AssertionError):") + + +class HarnessError(Exception): + pass + + +TARGETS = { + "pass not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", r'stubs("def a():\n pass\n")'), + "ellipsis not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", r'stubs("def a():\n ...\n")'), + "NotImplementedError not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", r"raise NotImplementedError\n"), + "raise call not unwrapped": ("tests.test_g11_census.PythonStubs.test_each_stub_body", "NotImplementedError('later')"), + "docstring-only not reported": ("tests.test_g11_census.PythonStubs.test_each_stub_body", "Only a docstring."), + "docstring not stripped": ("tests.test_g11_census.PythonStubs.test_docstring_then_stub_body", "Doc."), + "decorator context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), + "Protocol/ABC context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), + "exception context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), + "log_message context ignored": ("tests.test_g11_census.PythonStubs.test_request_handler_log_message_override_is_legitimate", "self.assertEqual(found,"), + "log_message context too broad": ("tests.test_g11_census.PythonStubs.test_request_handler_log_message_override_is_legitimate", "self.assertEqual(found,"), + "module docstring not recorded": ("tests.test_g11_census.ToolChecks.test_module_docstring_is_recorded", "self.assertEqual((report["), + "class bodies not visited": ("tests.test_g11_census.PythonStubs.test_methods_async_and_nested", "self.assertEqual(stubs(text),"), + "nested functions not visited": ("tests.test_g11_census.PythonStubs.test_methods_async_and_nested", "self.assertEqual(stubs(text),"), + "comments read as data": ("tests.test_g11_census.Markers.test_python_comment_is_marker_and_string_is_mention", "self.assertEqual(classes(run("), + "strings read as comments": ("tests.test_g11_census.Markers.test_python_comment_is_marker_and_string_is_mention", "self.assertEqual(classes(run("), + "marker words case-insensitive": ("tests.test_g11_census.Markers.test_lowercase_words_and_lookalikes_are_not_markers", "self.assertEqual(classes(run("), + "marker word boundary dropped": ("tests.test_g11_census.Markers.test_lowercase_words_and_lookalikes_are_not_markers", "self.assertEqual(classes(run("), + "quotes not tracked": ("tests.test_g11_census.Markers.test_javascript_comments_and_strings", "self.assertEqual(classes(run("), + "string escapes ignored": ("tests.test_g11_census.Markers.test_escaped_quote_inside_string", "self.assertEqual(classes(run("), + "hash comment needs no whitespace": ("tests.test_g11_census.Markers.test_shell_comments_and_strings", "self.assertEqual(classes(run("), + "block comments not entered": ("tests.test_g11_census.Markers.test_javascript_comments_and_strings", "self.assertEqual(classes(run("), + "block comment end ignored": ("tests.test_g11_census.Markers.test_javascript_comments_and_strings", "self.assertEqual(classes(run("), + "mentions dropped": ("tests.test_g11_census.Markers.test_python_comment_is_marker_and_string_is_mention", "self.assertEqual(classes(run("), + "noop detection off": ("tests.test_g11_census.Workflows.test_noop_steps", "self.assertEqual(found,"), + "block scalar not read": ("tests.test_g11_census.Workflows.test_noop_steps", "self.assertEqual(found,"), + "block scalar ends late": ("tests.test_g11_census.Workflows.test_noop_steps", "self.assertEqual(found,"), + "inline comment kept": ("tests.test_g11_census.Workflows.test_noop_steps", "self.assertEqual(found,"), + "tests counted as tools": ("tests.test_g11_census.ToolChecks.test_tests_and_init_files_are_not_tools", "self.assertEqual(sorted(report),"), + "__init__ counted as tool": ("tests.test_g11_census.ToolChecks.test_tests_and_init_files_are_not_tools", "self.assertEqual(sorted(report),"), + "basename ambiguity ignored": ("tests.test_g11_census.ToolChecks.test_ambiguous_basename_needs_the_full_path", 'self.assertFalse(report["tools/run.py"]["documented"])'), + "path reference not recognised": ("tests.test_g11_census.ToolChecks.test_full_path_counts_even_when_basename_is_ambiguous", 'self.assertTrue(report["tools/run.py"]["documented"])'), + "import not recognised": ("tests.test_g11_census.ToolChecks.test_documented_and_exercised_by_path_import_and_basename", 'self.assertTrue(report[path]["exercised"])'), + "from-import list not recognised": ("tests.test_g11_census.ToolChecks.test_documented_and_exercised_by_path_import_and_basename", 'self.assertTrue(report[path]["exercised"])'), + "basename not recognised": ("tests.test_g11_census.ToolChecks.test_documented_and_exercised_by_path_import_and_basename", 'self.assertTrue(report[path]["documented"])'), + "transitive exercise off": ("tests.test_g11_census.ToolChecks.test_exercised_through_an_exercised_tool", 'self.assertEqual(report["tools/helper.py"]["exercised_by"],'), + "site not production": ("tests.test_g11_census.Markers.test_site_marker_fails_and_vendor_marker_does_not", 'self.assertEqual(run({"docs/site/app.mjs":'), + "research counted as production": ("tests.test_g11_census.PythonStubs.test_production_stub_fails_but_research_stub_does_not", 'self.assertEqual(research["status"],'), + "vendor counted as site": ("tests.test_g11_census.Markers.test_site_marker_fails_and_vendor_marker_does_not", 'self.assertEqual(run({"docs/site/vendor/lib.js":'), + "undecodable decoded leniently": ("tests.test_g11_census.FailClosed.test_undecodable_and_unparseable_production_files_fail", 'self.assertEqual(report["status"], "FAIL")'), + "unparseable not a failure class": ("tests.test_g11_census.FailClosed.test_undecodable_and_unparseable_production_files_fail", 'self.assertEqual(report["status"], "FAIL")'), + "legitimate context still fails": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", 'self.assertEqual(report["failing"], [])'), + "markers do not fail": ("tests.test_g11_census.Markers.test_production_marker_fails", 'self.assertEqual(report["status"], "FAIL")'), + "noop steps do not fail": ("tests.test_g11_census.Workflows.test_production_noop_fails", 'self.assertEqual(run({".github/workflows/w.yml": " - run: true'), + "tool findings do not fail": ("tests.test_g11_census.ToolChecks.test_undocumented_and_unexercised_tools_fail", 'self.assertEqual(result["status"], "FAIL")'), + "exit status always zero": ("tests.test_g11_census.CommandLine.test_exit_codes_and_exact_committed_bytes", "self.assertEqual(result.returncode, 1)"), + "unreadable revision exits zero": ("tests.test_g11_census.CommandLine.test_exit_codes_and_exact_committed_bytes", 'self.assertEqual(self.cli(root, "--rev", "no-such-revision").returncode, 2)'), +} + + +NEW_MUTATIONS = [ + ("decoy decorator names accepted", "census", " if _trusted_decorator(dec, imports):", " if _decorator_name(dec) in LEGIT_DECORATORS:", + "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", 'self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]),'), + ("decoy abstract-base names accepted", "census", " if _trusted_abstract_base(base, imports):", " if _base_name(base) in LEGIT_BASES:", + "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", 'self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]),'), + ("decoy exception suffix accepted", "census", " if _builtin_exception_base(base, imports):", + ' if _base_name(base).endswith(("Error", "Exception", "Warning")):', + "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", 'self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]),'), + ("decoy request-handler suffix accepted", "census", ' if _http_request_handler_base(base, imports) and func.name == "log_message":', + ' if _base_name(base).endswith("RequestHandler") and func.name == "log_message":', + "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", 'self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]),'), + ("comment and string mentions count as exercise", "census", + "def python_tool_references(text, tools, unique):\n references = set()", + "def python_tool_references(text, tools, unique):\n references = {p for p in tools if p in text}", + "tests.test_g11_census.AdversarialControls.test_comment_and_string_mentions_do_not_exercise_tools", + 'self.assertFalse(report["tools/runner.py"]["exercised"])'), + ("empty workflow run block not detected", "census", " if not commands:", " if False:", + "tests.test_g11_census.AdversarialControls.test_empty_and_comment_only_run_blocks_fail", "self.assertEqual(found,"), + ("YAML comments omitted", "census", ' elif path.endswith((".yml", ".yaml")):', " elif False:", + "tests.test_g11_census.AdversarialControls.test_yaml_comments_are_scanned_for_markers", "self.assertEqual(classes(report),"), + ("shadowed builtin exception name accepted", "census", + 'elif isinstance(node, ast.Name) and node.id in builtins.__dict__ and node.id not in imports["shadowed"]:', + 'elif isinstance(node, ast.Name) and node.id in builtins.__dict__:', + "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", + 'self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]),'), + ("mutation attribution accepts wrong assertion", "harness", ' and outcome.get("traceback_line") == expected_line', " and True", + "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", + "self.assertFalse(g11m.qualifies_kill(wrong_line, expected, 17))"), +] + + +def _assertion_line(test_source, test_id, fragment): + parts = test_id.split(".") + class_name, method_name = parts[-2], parts[-1] + tree = ast.parse(test_source) + class_node = next((n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == class_name), None) + if class_node is None: + raise HarnessError("target class missing: " + class_name) + method = next((n for n in class_node.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) + and n.name == method_name), None) + if method is None: + raise HarnessError("target test missing: " + test_id) + matches = [] + for node in ast.walk(method): + if isinstance(node, ast.Call): + segment = ast.get_source_segment(test_source, node) or "" + if fragment in segment: + matches.append(node.lineno) + if len(matches) != 1: + raise HarnessError("assertion fragment %r matched %d calls in %s" % (fragment, len(matches), test_id)) + return matches[0] + + +def _parse_run(proc): + output = proc.stdout + "\n" + proc.stderr + summary = SUMMARY.search(output) + ran = int(summary.group(1)) if summary else -1 + result = {"exit": proc.returncode, "ran": ran, "failures": 0, "errors": 0, "skipped": 0, + "test_id": None, "traceback_method": None, "traceback_line": None, + "exception_type": None, "parsed": bool(summary)} + status = RESULT.findall(output) + if status: + result["status"] = status[-1][0] + detail = status[-1][1] or "" + counts = dict(re.findall(r"(failures|errors|skipped)=(\d+)", detail)) + result["failures"] = int(counts.get("failures", 0)) + result["errors"] = int(counts.get("errors", 0)) + result["skipped"] = int(counts.get("skipped", 0)) + header = FAIL_HEADER.search(output) + if header: + result["test_id"] = header.group(2) + frames = TRACE_FRAME.findall(output) + if frames: + _path, line, method = frames[-1] + result["traceback_line"] = int(line) + result["traceback_method"] = method + exc = EXCEPTION.search(output) + if exc: + result["exception_type"] = exc.group(1) + return result + + +def qualifies_kill(outcome, expected_test, expected_line): + expected_method = expected_test.rsplit(".", 1)[-1] + return ( + outcome.get("parsed") is True + and outcome.get("exit") == 1 + and outcome.get("ran") == 1 + and outcome.get("failures") == 1 + and outcome.get("errors") == 0 + and outcome.get("skipped") == 0 + and outcome.get("test_id") == expected_test + and outcome.get("traceback_method") == expected_method + and outcome.get("traceback_line") == expected_line + and outcome.get("exception_type") == "AssertionError" + ) + + +def _run(workdir, test_id=None): + command = [sys.executable, "-B", "-S"] + if sys.flags.optimize: + command.append("-O") + command.extend(["-m", "unittest"]) + command.append(test_id if test_id else "tests.test_g11_census") + command.append("-v") + proc = subprocess.run(command, cwd=workdir, capture_output=True, text=True, timeout=300) + return _parse_run(proc) + + +def _sandbox(tool_source, test_source, harness_source, workdir): + os.makedirs(os.path.join(workdir, "tests"), exist_ok=True) + os.makedirs(os.path.join(workdir, "tools", "cbz"), exist_ok=True) + Path(workdir, TESTS).write_text(test_source, encoding="utf-8") + Path(workdir, TOOL).write_text(tool_source, encoding="utf-8") + Path(workdir, HARNESS).write_text(harness_source, encoding="utf-8") + + +def run_selftest(root): + tool_path = os.path.join(root, TOOL) + test_path = os.path.join(root, TESTS) + harness_path = os.path.join(root, HARNESS) + with open(tool_path, encoding="utf-8") as fh: + original_tool = fh.read() + with open(test_path, encoding="utf-8") as fh: + test_source = fh.read() + with open(harness_path, encoding="utf-8") as fh: + original_harness = fh.read() + cases = [] + for name, old, new in MUTATIONS: + target_test, fragment = TARGETS[name] + cases.append((name, "census", old, new, target_test, fragment)) + cases.extend(NEW_MUTATIONS) + names = [case[0] for case in cases] + if len(names) != len(set(names)): + raise HarnessError("duplicate mutation names") + for name, target, old, new, test_id, fragment in cases: + source = original_tool if target == "census" else original_harness + if source.count(old) != 1 or old == new: + raise HarnessError("mutation %r: target occurs %d times in %s (expected exactly 1)" % + (name, source.count(old), target)) + _assertion_line(test_source, test_id, fragment) + baseline = None + results = [] + with tempfile.TemporaryDirectory(prefix="g11-mut-") as tmp: + base = os.path.join(tmp, "baseline") + _sandbox(original_tool, test_source, original_harness, base) + baseline = _run(base) + for index, (name, target, old, new, test_id, fragment) in enumerate(cases): + work = os.path.join(tmp, "m%02d" % index) + tool_source = original_tool + harness_source = original_harness + if target == "census": + tool_source = original_tool.replace(old, new, 1) + else: + harness_source = original_harness.replace(old, new, 1) + _sandbox(tool_source, test_source, harness_source, work) + outcome = _run(work, test_id) + expected_line = _assertion_line(test_source, test_id, fragment) + results.append(dict(outcome, mutation=name, target=target, expected_test=test_id, + expected_line=expected_line, + killed=qualifies_kill(outcome, test_id, expected_line))) + baseline_ok = (baseline.get("parsed") and baseline.get("exit") == 0 and baseline.get("ran") == 33 + and baseline.get("failures") == 0 and baseline.get("errors") == 0 + and baseline.get("skipped") == 0 and baseline.get("status") == "OK") + killed = sum(r["killed"] for r in results) + return {"baseline": baseline, "mutations": results, "killed": killed, "total": len(results), + "status": "PASS" if baseline_ok and killed == len(results) else "FAIL"} + + +def main(argv=None): + parser = argparse.ArgumentParser(description="Mutation self-test of the G11 census and its kill-attribution guard.") + parser.add_argument("--root", default=".") + parser.add_argument("--json", action="store_true") + args = parser.parse_args(argv) + try: + report = run_selftest(os.path.abspath(args.root)) + except (HarnessError, OSError, subprocess.SubprocessError) as exc: + print("G11_SELFTEST_HARNESS_ERROR: %s" % exc, file=sys.stderr) + return 2 + if args.json: + print(json.dumps(report, indent=1, sort_keys=True)) + else: + for r in report["mutations"]: + print("%-8s %s (target %s, exit %d, failures %d, errors %d, skips %d, line %s/%s)" % + ("KILLED" if r["killed"] else "SURVIVED", r["mutation"], r["target"], r["exit"], + r["failures"], r["errors"], r["skipped"], r["traceback_line"], r["expected_line"])) + print("G11_SELFTEST_%s: baseline %d tests, %d/%d mutations killed" % + (report["status"], report["baseline"].get("ran", -1), report["killed"], report["total"])) + return 0 if report["status"] == "PASS" else 1 + + +if __name__ == "__main__": + sys.exit(main()) From fcaac866bbef69cf95d2ab8c1220f0abf9a30e1e Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:31:09 -0500 Subject: [PATCH 04/27] CBZ-0 G11: document bounded auditor scope --- docs/cbz/G11_AUDITOR.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 docs/cbz/G11_AUDITOR.md diff --git a/docs/cbz/G11_AUDITOR.md b/docs/cbz/G11_AUDITOR.md new file mode 100644 index 000000000..4d3ea0bc7 --- /dev/null +++ b/docs/cbz/G11_AUDITOR.md @@ -0,0 +1,30 @@ +# G11 auditor — bounded census, not certification + +## Purpose + +`tools/cbz/g11_census.py` reads tracked blob bytes from one Git revision and reports review triggers for the owner-defined G11 tooling-hygiene criterion: Python stubs, code-comment markers, no-op workflow run steps, and tool documentation/exercise references. It is a triage instrument, not a proof, a Layer A certificate, or a federation-wide G11 admission. + +## Commands + +From the repository root, using only the Python standard library: + +```sh +python3 -B -S -m unittest tests.test_g11_census +python3 -B -S -O -m unittest tests.test_g11_census +python3 -B -S tools/cbz/g11_mutation_selftest.py +python3 -B -S -O tools/cbz/g11_mutation_selftest.py +python3 -B -S tools/cbz/g11_census.py --rev --json census.json +``` + +The census exits 0 for no detected production finding, 1 for a detected production finding, and 2 when the requested Git revision cannot be read. A zero exit is only the result of this implementation over its stated scanned paths and rules. + +## Soundness boundaries + +- Stub exemptions are bound to imported `abc.abstractmethod`, `typing.overload`, `typing_extensions.overload`, `abc.ABC`, `typing.Protocol`, `typing_extensions.Protocol`, actual built-in exception classes, and the named `http.server` request-handler bases. Name suffixes and locally defined look-alikes do not grant exemptions. +- Python test/tool exercise evidence uses parsed imports and recognized subprocess invocations, not arbitrary comments or string mentions. JavaScript test imports are read from import declarations after comment stripping. Workflow exercise evidence is read from run commands, not YAML comments. +- YAML comments are scanned for marker words. Empty and comment-only `run` blocks are reported as no-op steps. +- The mutation self-test preserves the original mutation inventory and admits a kill only when the clean full 33-test baseline succeeds and a single designated test body fails at its exact designated assertion line with one AssertionError, zero errors, and zero skips. The harness propagates the parent's optimized mode to child tests and also has a mutation against its own attribution predicate. + +## Limits + +This scanner does not establish Python runtime behavior, shell/YAML semantics in general, JavaScript parsing correctness, completeness of every possible test/execution mechanism, or semantic absence of placeholders introduced by metaprogramming. The comment lexer for JavaScript/shell is intentionally conservative. Findings and misses require source-aware review. A green component workflow is not G11 gate acceptance, Layer A compliance, proof closure, scientific acceptance, or CBZ-0. From 651ac44ee1bae252e30ea7b1b1d7294fe5ba37e5 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:31:12 -0500 Subject: [PATCH 05/27] CBZ-0 G11: add pinned component self-test workflow --- .github/workflows/cbz-g11-auditor.yml | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/cbz-g11-auditor.yml diff --git a/.github/workflows/cbz-g11-auditor.yml b/.github/workflows/cbz-g11-auditor.yml new file mode 100644 index 000000000..0c0dbce27 --- /dev/null +++ b/.github/workflows/cbz-g11-auditor.yml @@ -0,0 +1,28 @@ +name: CBZ G11 auditor component self-tests + +on: + pull_request: + push: + +permissions: + contents: read + +jobs: + g11-auditor: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: '3.11' + - name: Run census controls normally + run: python3 -B -S -m unittest tests.test_g11_census + - name: Run census controls with optimization + run: python3 -B -S -O -m unittest tests.test_g11_census + - name: Run mutation self-test normally + run: python3 -B -S tools/cbz/g11_mutation_selftest.py + - name: Run mutation self-test with optimization + run: python3 -B -S -O tools/cbz/g11_mutation_selftest.py From 5ad42e778509e00164718e5000def526fd0cb382 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:44:29 -0500 Subject: [PATCH 06/27] CBZ-0 G11: bind stub exemptions and exercise evidence to actual scopes --- tools/cbz/g11_census.py | 81 +++++++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 11 deletions(-) diff --git a/tools/cbz/g11_census.py b/tools/cbz/g11_census.py index ee2f2bc05..5463204ad 100644 --- a/tools/cbz/g11_census.py +++ b/tools/cbz/g11_census.py @@ -106,6 +106,42 @@ def _target_names(node): return set() +def _scope_bindings(scope): + """Names bound in one lexical scope; nested scope bodies are kept separate.""" + names = set() + if isinstance(scope, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + args = scope.args + for arg in args.posonlyargs + args.args + args.kwonlyargs: + names.add(arg.arg) + if args.vararg: + names.add(args.vararg.arg) + if args.kwarg: + names.add(args.kwarg.arg) + + def visit(node, root=False): + if not root and isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(node.name) + return + if not root and isinstance(node, ast.Lambda): + return + if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)): + names.add(node.id) + elif isinstance(node, ast.Import): + for alias in node.names: + names.add(alias.asname or alias.name.split(".", 1)[0]) + elif isinstance(node, ast.ImportFrom): + for alias in node.names: + if alias.name != "*": + names.add(alias.asname or alias.name) + elif isinstance(node, ast.ExceptHandler) and node.name: + names.add(node.name) + for child in ast.iter_child_nodes(node): + visit(child) + + visit(scope, True) + return names + + def _import_bindings(tree): modules, symbols, shadowed = {}, {}, set() for node in tree.body: @@ -147,8 +183,15 @@ def _resolve_import(node, imports): return tuple(resolved.rsplit(".", 1)) -def _trusted_decorator(node, imports): - resolved = _resolve_import(node, imports) +def _trusted_decorator(node, imports, scope_shadowed=()): + target = node + while isinstance(target, ast.Call): + target = target.func + dotted = _dotted_name(target) + binding = dotted.split(".", 1)[0] if dotted else "" + if binding and binding in scope_shadowed: + return False + resolved = _resolve_import(target, imports) return resolved in {("abc", "abstractmethod"), ("typing", "overload"), ("typing_extensions", "overload")} @@ -203,9 +246,9 @@ def _stub_kind(body): return None -def _legit_context(func, classes, imports): +def _legit_context(func, classes, imports, scope_shadowed=()): for dec in func.decorator_list: - if _trusted_decorator(dec, imports): + if _trusted_decorator(dec, imports, scope_shadowed): return "@" + _decorator_name(dec) for cls in reversed(classes): for base in cls.bases: @@ -224,19 +267,19 @@ def python_stubs(tree, imports=None): if imports is None: imports = _import_bindings(tree) - def visit(node, classes): + def visit(node, classes, scope_shadowed): for child in ast.iter_child_nodes(node): if isinstance(child, ast.ClassDef): - visit(child, classes + [child]) + visit(child, classes + [child], scope_shadowed | _scope_bindings(child)) elif isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): kind = _stub_kind(child.body) if kind: found.append({"line": child.lineno, "name": child.name, "kind": kind, - "context": _legit_context(child, classes, imports)}) - visit(child, []) + "context": _legit_context(child, classes, imports, scope_shadowed)}) + visit(child, [], scope_shadowed | _scope_bindings(child)) else: - visit(child, classes) - visit(tree, []) + visit(child, classes, scope_shadowed) + visit(tree, [], set()) return found @@ -481,6 +524,9 @@ def python_tool_references(text, tools, unique): else: tokens = [] if tokens and tokens[0] in {"python", "python3", "python3.11", "bash", "sh"}: + # Inline -c payloads are code strings, not proof that a path was executed. + if "-c" in tokens[1:]: + continue for token in tokens[1:]: if token == "-m": continue @@ -522,6 +568,16 @@ def _strip_js_comments(text): i += 1 continue if quote: + if quote == "`": + if ch == "\\" and i + 1 < len(text): + out.extend((" ", " ")) + i += 2 + continue + if ch == quote: + quote = None + out.append("\n" if ch == "\n" else " ") + i += 1 + continue out.append(ch) if ch == "\\" and i + 1 < len(text): out.append(text[i + 1]) @@ -533,7 +589,7 @@ def _strip_js_comments(text): continue if ch in "\"'" + chr(96): quote = ch - out.append(ch) + out.append(" " if ch == chr(96) else ch) i += 1 continue if ch == "/" and nxt == "/": @@ -577,6 +633,9 @@ def workflow_tool_references(text, tools): if not tokens: continue if tokens[0] in {"python", "python3", "python3.11", "bash", "sh"}: + # Do not treat a path mentioned inside an inline -c program as an invocation. + if "-c" in tokens[1:]: + continue for index, token in enumerate(tokens[1:], 1): if token == "-m" and index + 1 < len(tokens): module_path = tokens[index + 1].replace(".", "/") + ".py" From 385b0384d46dfcc7924412bf7cad7be5c09851ea Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:45:22 -0500 Subject: [PATCH 07/27] CBZ-0 G11: add adversarial false-exercise and scope-shadow controls --- tests/test_g11_census.py | 61 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/tests/test_g11_census.py b/tests/test_g11_census.py index 54ecc91b7..0349830ce 100644 --- a/tests/test_g11_census.py +++ b/tests/test_g11_census.py @@ -297,6 +297,17 @@ def test_decoy_names_do_not_exempt_stubs(self): self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]), ["abc", "builtin", "decoy_decorator", "decoy_qualified", "fail", "fail", "fail", "log_message", "proto"]) self.assertEqual(report["status"], "FAIL") + local_shadow = ( + "from abc import abstractmethod\n" + "def factory():\n" + " abstractmethod = lambda fn: fn\n" + " @abstractmethod\n" + " def local_decoy():\n" + " pass\n" + ) + local_report = run({"tools/good.py": local_shadow, **DOCUMENTED_TESTED}) + self.assertEqual([(f["class"], f["detail"]) for f in local_report["findings"] + if f["class"].startswith("stub")], [("stub", "local_decoy: pass")]) def test_comment_and_string_mentions_do_not_exercise_tools(self): tree = { @@ -305,9 +316,30 @@ def test_comment_and_string_mentions_do_not_exercise_tools(self): "tests/test_runner.py": "# tools/runner.py\nDOC = 'tools/runner.py'\nprint('tools/runner.py')\n", } report = run(tree) - self.assertFalse(report["tools/runner.py"]["exercised"]) + self.assertFalse(report["tools"]["tools/runner.py"]["exercised"]) self.assertEqual(report["failing_tools"], ["tools/runner.py"]) + python_cmd = run({ + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + "tests/test_runner.py": 'import subprocess\nsubprocess.run("python -c \\'tools/runner.py\\'", shell=True)\n', + }) + self.assertFalse(python_cmd["tools"]["tools/runner.py"]["exercised"]) + + shell_cmd = run({ + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + ".github/workflows/ci.yml": "jobs:\n test:\n steps:\n - run: sh -c 'echo tools/runner.py'\n", + }) + self.assertFalse(shell_cmd["tools"]["tools/runner.py"]["exercised"]) + + js_cmd = run({ + "tools/runner.mjs": "export const run = () => 1;\n", + "tools/README.md": "Use tools/runner.mjs.\n", + "tests/runner.test.mjs": "const sample = \`\nimport '../tools/runner.mjs';\n\`;\n", + }) + self.assertFalse(js_cmd["tools"]["tools/runner.mjs"]["exercised"]) + def test_empty_and_comment_only_run_blocks_fail(self): text = ("jobs:\n a:\n steps:\n" " - run: |\n # only comment\n" @@ -328,12 +360,35 @@ def test_yaml_comments_are_scanned_for_markers(self): class MutationAttribution(unittest.TestCase): def test_mutation_kill_requires_exact_target_assertion(self): expected = "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion" - good = {"exit": 1, "ran": 1, "failures": 1, "errors": 0, "skipped": 0, - "test_id": expected, "traceback_method": "test_mutation_kill_requires_exact_target_assertion", + good = {"parsed": True, "status": "FAILED", "exit": 1, "ran": 1, "failures": 1, + "errors": 0, "skipped": 0, "test_id": expected, + "traceback_method": "test_mutation_kill_requires_exact_target_assertion", "traceback_line": 17, "exception_type": "AssertionError"} self.assertTrue(g11m.qualifies_kill(good, expected, 17)) wrong_line = dict(good, traceback_line=18) self.assertFalse(g11m.qualifies_kill(wrong_line, expected, 17)) + wrong_test = dict(good, test_id=expected + ".other") + self.assertFalse(g11m.qualifies_kill(wrong_test, expected, 17)) + not_failed = dict(good, status="OK") + self.assertFalse(g11m.qualifies_kill(not_failed, expected, 17)) + has_errors = dict(good, errors=1) + self.assertFalse(g11m.qualifies_kill(has_errors, expected, 17)) + has_skips = dict(good, skipped=1) + self.assertFalse(g11m.qualifies_kill(has_skips, expected, 17)) + + expected_parser = "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion" + output = ( + "Ran 1 test in 0.001s\n\n" + "FAIL: test_mutation_kill_requires_exact_target_assertion (tests.test_g11_census.MutationAttribution)\n" + "Traceback (most recent call last):\n" + ' File "/tmp/tests/test_g11_census.py", line 123, in test_mutation_kill_requires_exact_target_assertion\n' + "AssertionError: expected\n\n" + "FAILED (failures=1)\n" + ) + proc = subprocess.CompletedProcess(["python"], 1, output, "") + parsed = g11m._parse_run(proc) + self.assertEqual(parsed["test_id"], expected_parser) + self.assertTrue(g11m.qualifies_kill(parsed, expected_parser, 123)) if __name__ == "__main__": From 8602ddc572e0037fc43f90bf1aeb17b7fe976d91 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:49:17 -0500 Subject: [PATCH 08/27] CBZ-0 G11: bind mutation kills to exact unittest failures --- tools/cbz/g11_mutation_selftest.py | 69 +++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 10 deletions(-) diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py index 828864100..d050157e2 100644 --- a/tools/cbz/g11_mutation_selftest.py +++ b/tools/cbz/g11_mutation_selftest.py @@ -1,12 +1,15 @@ #!/usr/bin/env python3 """Mutation self-test of the G11 census (tools/cbz/g11_census.py). -Each mutation weakens the census in one way. For every mutation the harness -copies the census and its test module into an isolated directory, applies the -mutation, and runs the test module there. A mutation is KILLED only when the -test run reports at least one assertion failure and no error, so a mutant that -breaks the import or crashes the tests does not count. The self-test passes -only when the unmutated copy passes and every mutation is killed. +Each mutation weakens the census or its attribution guard in one way. For +every mutation the harness copies the census, test module and harness into an +isolated directory, applies exactly one mutation, and runs one designated test. +A mutation is KILLED only when the clean full baseline passes and that one test +reports exactly one AssertionError at its exact designated assertion line, +with the exact module/class/method identity, FAILED status, zero errors and +zero skips. Setup, import, teardown, cleanup, process and unrelated assertion +failures never count as kills. The self-test passes only when the unmutated +baseline passes and every mutation is killed. A mutation whose target text does not occur exactly once is a harness error, never a silent no-op. @@ -41,7 +44,7 @@ ("docstring-only not reported", ' if not rest:\n return "docstring only"\n', " if not rest:\n return None\n"), ("docstring not stripped", " rest = rest[1:]\n", " rest = rest[0:]\n"), - ("decorator context ignored", " if _trusted_decorator(dec, imports):", " if False:"), + ("decorator context ignored", " if _trusted_decorator(dec, imports, scope_shadowed):", " if False:"), ("Protocol/ABC context ignored", " if _trusted_abstract_base(base, imports):", " if False:"), ("exception context ignored", " if _builtin_exception_base(base, imports):", " if False:"), ("log_message context ignored", ' if _http_request_handler_base(base, imports) and func.name == "log_message":', " if False:"), @@ -172,7 +175,7 @@ class HarnessError(Exception): NEW_MUTATIONS = [ - ("decoy decorator names accepted", "census", " if _trusted_decorator(dec, imports):", " if _decorator_name(dec) in LEGIT_DECORATORS:", + ("decoy decorator names accepted", "census", " if _trusted_decorator(dec, imports, scope_shadowed):", " if _decorator_name(dec) in LEGIT_DECORATORS:", "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", 'self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]),'), ("decoy abstract-base names accepted", "census", " if _trusted_abstract_base(base, imports):", " if _base_name(base) in LEGIT_BASES:", "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", 'self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]),'), @@ -186,7 +189,7 @@ class HarnessError(Exception): "def python_tool_references(text, tools, unique):\n references = set()", "def python_tool_references(text, tools, unique):\n references = {p for p in tools if p in text}", "tests.test_g11_census.AdversarialControls.test_comment_and_string_mentions_do_not_exercise_tools", - 'self.assertFalse(report["tools/runner.py"]["exercised"])'), + 'self.assertFalse(report["tools"]["tools/runner.py"]["exercised"])'), ("empty workflow run block not detected", "census", " if not commands:", " if False:", "tests.test_g11_census.AdversarialControls.test_empty_and_comment_only_run_blocks_fail", "self.assertEqual(found,"), ("YAML comments omitted", "census", ' elif path.endswith((".yml", ".yaml")):', " elif False:", @@ -199,6 +202,51 @@ class HarnessError(Exception): ("mutation attribution accepts wrong assertion", "harness", ' and outcome.get("traceback_line") == expected_line', " and True", "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", "self.assertFalse(g11m.qualifies_kill(wrong_line, expected, 17))"), + ("function-local decorator shadow accepted", "census", + " if binding and binding in scope_shadowed:\n return False", + " if False:\n return False", + "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", + 'self.assertEqual([(f["class"], f["detail"]) for f in local_report["findings"]'), + ("python -c path mention counts as execution", "census", + ' if "-c" in tokens[1:]:\n continue', + " if False:\n continue", + "tests.test_g11_census.AdversarialControls.test_comment_and_string_mentions_do_not_exercise_tools", + 'self.assertFalse(python_cmd["tools"]["tools/runner.py"]["exercised"])'), + ("shell -c path mention counts as execution", "census", + ' if "-c" in tokens[1:]:\n continue', + " if False:\n continue", + "tests.test_g11_census.AdversarialControls.test_comment_and_string_mentions_do_not_exercise_tools", + 'self.assertFalse(shell_cmd["tools"]["tools/runner.py"]["exercised"])'), + ("template literal import text counts as static import", "census", + ' if quote == "`":\n if ch == "\\\\" and i + 1 < len(text):\n out.extend((" ", " "))\n i += 2\n continue\n if ch == quote:\n quote = None\n out.append("\\n" if ch == "\\n" else " ")\n i += 1\n continue', + ' if quote == "`":\n if ch == "\\\\" and i + 1 < len(text):\n out.extend((" ", " "))\n i += 2\n continue\n if ch == quote:\n quote = None\n out.append(ch)\n i += 1\n continue', + "tests.test_g11_census.AdversarialControls.test_comment_and_string_mentions_do_not_exercise_tools", + 'self.assertFalse(js_cmd["tools"]["tools/runner.mjs"]["exercised"])'), + ("parser drops method identity", "harness", + ' result["test_id"] = header.group(2) + "." + header.group(1)', + ' result["test_id"] = header.group(2)', + "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", + 'self.assertEqual(parsed["test_id"], expected_parser)'), + ("wrong test id credited", "harness", + ' and outcome.get("test_id") == expected_test', + " and True", + "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", + "self.assertFalse(g11m.qualifies_kill(wrong_test, expected, 17))"), + ("nonfailure status credited", "harness", + ' and outcome.get("status") == "FAILED"', + " and True", + "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", + "self.assertFalse(g11m.qualifies_kill(not_failed, expected, 17))"), + ("error outcome credited", "harness", + ' and outcome.get("errors") == 0', + " and True", + "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", + "self.assertFalse(g11m.qualifies_kill(has_errors, expected, 17))"), + ("skipped outcome credited", "harness", + ' and outcome.get("skipped") == 0', + " and True", + "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", + "self.assertFalse(g11m.qualifies_kill(has_skips, expected, 17))"), ] @@ -241,7 +289,7 @@ def _parse_run(proc): result["skipped"] = int(counts.get("skipped", 0)) header = FAIL_HEADER.search(output) if header: - result["test_id"] = header.group(2) + result["test_id"] = header.group(2) + "." + header.group(1) frames = TRACE_FRAME.findall(output) if frames: _path, line, method = frames[-1] @@ -257,6 +305,7 @@ def qualifies_kill(outcome, expected_test, expected_line): expected_method = expected_test.rsplit(".", 1)[-1] return ( outcome.get("parsed") is True + and outcome.get("status") == "FAILED" and outcome.get("exit") == 1 and outcome.get("ran") == 1 and outcome.get("failures") == 1 From f47d06934b073b6b5c4bf616610c76b808d41237 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:49:34 -0500 Subject: [PATCH 09/27] CBZ-0 G11: document conservative command and import evidence boundaries --- docs/cbz/G11_AUDITOR.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/cbz/G11_AUDITOR.md b/docs/cbz/G11_AUDITOR.md index 4d3ea0bc7..763f78eb6 100644 --- a/docs/cbz/G11_AUDITOR.md +++ b/docs/cbz/G11_AUDITOR.md @@ -20,10 +20,10 @@ The census exits 0 for no detected production finding, 1 for a detected producti ## Soundness boundaries -- Stub exemptions are bound to imported `abc.abstractmethod`, `typing.overload`, `typing_extensions.overload`, `abc.ABC`, `typing.Protocol`, `typing_extensions.Protocol`, actual built-in exception classes, and the named `http.server` request-handler bases. Name suffixes and locally defined look-alikes do not grant exemptions. -- Python test/tool exercise evidence uses parsed imports and recognized subprocess invocations, not arbitrary comments or string mentions. JavaScript test imports are read from import declarations after comment stripping. Workflow exercise evidence is read from run commands, not YAML comments. +- Stub exemptions are bound to imported `abc.abstractmethod`, `typing.overload`, `typing_extensions.overload`, `abc.ABC`, `typing.Protocol`, `typing_extensions.Protocol`, actual built-in exception classes, and the named `http.server` request-handler bases. Name suffixes, locally defined look-alikes, and enclosing function/class bindings that shadow an imported decorator do not grant exemptions. +- Python test/tool exercise evidence uses parsed imports and recognized subprocess invocations, not arbitrary comments or string mentions. Python and shell `-c` payloads are deliberately not credited from path mentions alone. JavaScript test imports are read from static import declarations after comment and template-literal masking; dynamic imports are outside this recognizer. Workflow exercise evidence is read from run commands, not YAML comments. - YAML comments are scanned for marker words. Empty and comment-only `run` blocks are reported as no-op steps. -- The mutation self-test preserves the original mutation inventory and admits a kill only when the clean full 33-test baseline succeeds and a single designated test body fails at its exact designated assertion line with one AssertionError, zero errors, and zero skips. The harness propagates the parent's optimized mode to child tests and also has a mutation against its own attribution predicate. +- The mutation self-test preserves the original mutation inventory and admits a kill only when the clean full 33-test baseline succeeds and the exact designated module/class/method reports `FAILED` with one AssertionError at its exact designated assertion line, zero errors, and zero skips. The harness propagates the parent's optimized mode to child tests and mutates its own parser and each attribution guard. ## Limits From e80f3276950936764be4f88077ce15a2fe3dacb4 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:55:40 -0500 Subject: [PATCH 10/27] CBZ-0 G11: preserve quoted workflow command boundaries --- tools/cbz/g11_census.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/cbz/g11_census.py b/tools/cbz/g11_census.py index 5463204ad..80a8496f0 100644 --- a/tools/cbz/g11_census.py +++ b/tools/cbz/g11_census.py @@ -419,7 +419,9 @@ def _workflow_commands(text): else: commands = [] else: - commands = [re.sub(r"\s+#.*$", "", value).strip("'\"")] + if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"": + value = value[1:-1] + commands = [_strip_shell_comment(value).strip()] commands = [_strip_shell_comment(c).strip() for c in commands] commands = [c for c in commands if c] found.append((number, commands)) From 179b7776e79c751ca9c43bea4cb98bdb2a9b7197 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:55:51 -0500 Subject: [PATCH 11/27] CBZ-0 G11: exercise valid shell command quoting and template literals --- tests/test_g11_census.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_g11_census.py b/tests/test_g11_census.py index 0349830ce..9eec48ed8 100644 --- a/tests/test_g11_census.py +++ b/tests/test_g11_census.py @@ -326,17 +326,19 @@ def test_comment_and_string_mentions_do_not_exercise_tools(self): }) self.assertFalse(python_cmd["tools"]["tools/runner.py"]["exercised"]) + workflow_text = "jobs:\n test:\n steps:\n - run: \"sh -c 'echo tools/runner.py'\"\n" shell_cmd = run({ "tools/runner.py": "def main(): return 0\n", "tools/README.md": "Use tools/runner.py.\n", - ".github/workflows/ci.yml": "jobs:\n test:\n steps:\n - run: sh -c 'echo tools/runner.py'\n", + ".github/workflows/ci.yml": workflow_text, }) + self.assertEqual(g11._workflow_commands(workflow_text)[0][1], ["sh -c 'echo tools/runner.py'"]) self.assertFalse(shell_cmd["tools"]["tools/runner.py"]["exercised"]) js_cmd = run({ "tools/runner.mjs": "export const run = () => 1;\n", "tools/README.md": "Use tools/runner.mjs.\n", - "tests/runner.test.mjs": "const sample = \`\nimport '../tools/runner.mjs';\n\`;\n", + "tests/runner.test.mjs": "const sample = `\nimport '../tools/runner.mjs';\n`;\n", }) self.assertFalse(js_cmd["tools"]["tools/runner.mjs"]["exercised"]) From 93a9807d862fb896e09163cf51151f87c625611d Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:56:38 -0500 Subject: [PATCH 12/27] CBZ-0 G11: keep mutation cases aligned with scope and workflow parsers --- tools/cbz/g11_mutation_selftest.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py index d050157e2..0ae8ecb53 100644 --- a/tools/cbz/g11_mutation_selftest.py +++ b/tools/cbz/g11_mutation_selftest.py @@ -52,8 +52,8 @@ ' if _http_request_handler_base(base, imports):'), ("module docstring not recorded", " docstring = bool(ast.get_docstring(ast.parse(files[path])))", " docstring = True"), - ("class bodies not visited", " visit(child, classes + [child])", " pass"), - ("nested functions not visited", " visit(child, [])\n else:", + ("class bodies not visited", " visit(child, classes + [child], scope_shadowed | _scope_bindings(child))", " pass"), + ("nested functions not visited", " visit(child, [], scope_shadowed | _scope_bindings(child))\n else:", " pass\n else:"), ("comments read as data", " if tok.type == tokenize.COMMENT:\n target = markers", " if tok.type == tokenize.COMMENT:\n target = mentions"), @@ -77,8 +77,8 @@ ("block scalar not read", ' if not value or value[0] in "|>":', " if False:"), ("block scalar ends late", " if nxt.strip() and len(nxt) - len(nxt.lstrip()) <= key_column:", " if nxt.strip() and len(nxt) - len(nxt.lstrip()) < key_column:"), - ("inline comment kept", ' commands = [re.sub(r"\\s+#.*$", "", value).strip("\'\\"")]', - ' commands = [value.strip("\'\\"")]'), + ("inline comment kept", " commands = [_strip_shell_comment(value).strip()]", + " commands = [value.strip()]"), ("tests counted as tools", ' and area(p) == "tooling" and not p.endswith("__init__.py"))', ' and True and not p.endswith("__init__.py"))'), ("__init__ counted as tool", ' and area(p) == "tooling" and not p.endswith("__init__.py"))', @@ -247,6 +247,11 @@ class HarnessError(Exception): " and True", "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", "self.assertFalse(g11m.qualifies_kill(has_skips, expected, 17))"), + ("quoted workflow command boundary ignored", "census", + " value = value[1:-1]", + " value = value", + "tests.test_g11_census.AdversarialControls.test_comment_and_string_mentions_do_not_exercise_tools", + "self.assertEqual(g11._workflow_commands(workflow_text)[0][1], [\"sh -c 'echo tools/runner.py'\"])"), ] From 6037ed3c621bb708b2386346d00d6891019a2e8b Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:00:56 -0500 Subject: [PATCH 13/27] CBZ-0 G11: exclude unittest failure headers from exception attribution --- tools/cbz/g11_mutation_selftest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py index 0ae8ecb53..e9e9dce76 100644 --- a/tools/cbz/g11_mutation_selftest.py +++ b/tools/cbz/g11_mutation_selftest.py @@ -116,7 +116,7 @@ SUMMARY = re.compile(r"Ran (\d+) tests? in [^\n]+") FAIL_HEADER = re.compile(r"^FAIL: ([^(]+) \(([^)]+)\)$", re.M) TRACE_FRAME = re.compile(r'File "([^"]*test_g11_census\.py)", line (\d+), in ([A-Za-z_]\w*)') -EXCEPTION = re.compile(r"(?m)^(AssertionError):") +EXCEPTION = re.compile(r"(?m)^([A-Za-z_][A-Za-z0-9_]*(?:Error|Exception|Warning|Interrupt|Exit|StopIteration)):") class HarnessError(Exception): @@ -252,6 +252,11 @@ class HarnessError(Exception): " value = value", "tests.test_g11_census.AdversarialControls.test_comment_and_string_mentions_do_not_exercise_tools", "self.assertEqual(g11._workflow_commands(workflow_text)[0][1], [\"sh -c 'echo tools/runner.py'\"])"), + ("FAIL header mistaken for exception", "harness", + 'EXCEPTION = re.compile(r"(?m)^([A-Za-z_][A-Za-z0-9_]*):")', + 'EXCEPTION = re.compile(r"(?m)^([A-Za-z_][A-Za-z0-9_]*(?:Error|Exception|Warning|Interrupt|Exit|StopIteration)):")', + "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", + "self.assertTrue(g11m.qualifies_kill(parsed, expected_parser, 123))"), ] From a3a56bd894cd642dc9cadc7d0a6b8c5d8cd34f8a Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:03:53 -0500 Subject: [PATCH 14/27] test(cbz): expose residual G11 auditor false positives --- tests/test_g11_census.py | 159 +++++++++++++++++++++++++++-- tools/cbz/g11_mutation_selftest.py | 2 +- 2 files changed, 149 insertions(+), 12 deletions(-) diff --git a/tests/test_g11_census.py b/tests/test_g11_census.py index 9eec48ed8..701537333 100644 --- a/tests/test_g11_census.py +++ b/tests/test_g11_census.py @@ -77,19 +77,18 @@ def test_real_bodies_are_not_stubs(self): self.assertEqual(stubs(text), []) def test_legitimate_contexts_are_reported_but_do_not_fail(self): - text = ("from abc import ABC, abstractmethod\nfrom typing import Protocol, overload\n" - "class Base(ABC):\n def m(self):\n ...\n" - "class Shape(Protocol):\n def area(self) -> float:\n ...\n" - "class Bad(ValueError):\n def __init__(self):\n pass\n" - "class C:\n @abstractmethod\n def a(self):\n pass\n" - " @overload\n def f(self, x: int) -> int:\n ...\n") + text = ("from abc import ABC, abstractmethod\\nfrom typing import Protocol, overload\\n" + "class Base(ABC):\\n def m(self):\\n ...\\n" + "class Shape(Protocol):\\n def area(self) -> float:\\n ...\\n" + "class Bad(ValueError):\\n def __init__(self):\\n pass\\n" + "class C:\\n @abstractmethod\\n def a(self):\\n pass\\n" + " @overload\\n def f(self, x: int) -> int:\\n ...\\n") report = run({"tools/good.py": text, **DOCUMENTED_TESTED}) contexts = sorted(f["detail"] for f in report["findings"] if f["class"] == "stub-context") - self.assertEqual(contexts, ["__init__: pass; method of an exception class", - "a: pass; @abstractmethod", "area: ellipsis; method of a Protocol class", - "f: ellipsis; @overload", "m: ellipsis; method of a ABC class"]) - self.assertEqual(report["failing"], []) - self.assertEqual(report["status"], "PASS") + self.assertEqual(contexts, ["a: pass; @abstractmethod", + "area: ellipsis; method of a Protocol class", "f: ellipsis; @overload"]) + self.assertEqual(sorted(f["detail"] for f in report["failing"]), ["__init__: pass", "m: ellipsis"]) + self.assertEqual(report["status"], "FAIL") def test_request_handler_log_message_override_is_legitimate(self): text = ("from http.server import SimpleHTTPRequestHandler\n" @@ -393,5 +392,143 @@ def test_mutation_kill_requires_exact_target_assertion(self): self.assertTrue(g11m.qualifies_kill(parsed, expected_parser, 123)) + + + +class ReviewResidualControls(unittest.TestCase): + def test_stub_exemptions_are_immediate_and_scope_safe(self): + text = ( + "from abc import ABC, abstractmethod\\n" + "from typing import Protocol\\n" + "from http.server import BaseHTTPRequestHandler\\n" + "class Abstract(ABC):\\n" + " @abstractmethod\\n" + " def required(self): ...\\n" + " def concrete(self): ...\\n" + " class Nested:\\n" + " def inner_abc(self): ...\\n" + "class Interface(Protocol):\\n" + " def declared(self): ...\\n" + " class Nested:\\n" + " def inner_protocol(self): ...\\n" + "class Failure(Exception):\\n" + " def exception_method(self): ...\\n" + "def build(ABC, Exception, BaseHTTPRequestHandler):\\n" + " class ShadowedABC(ABC):\\n" + " def shadowed_abc(self): ...\\n" + " class ShadowedError(Exception):\\n" + " def shadowed_exception(self): ...\\n" + " class ShadowedHandler(BaseHTTPRequestHandler):\\n" + " def shadowed_handler_log_message(self, *args): ...\\n" + ) + report = run({"tools/good.py": text, **DOCUMENTED_TESTED}) + stub_names = {f["detail"].split(":")[0] for f in report["findings"] if f["class"] == "stub"} + context_names = {f["detail"].split(":")[0] for f in report["findings"] if f["class"] == "stub-context"} + self.assertIn("required", context_names) + self.assertIn("declared", context_names) + for name in ("concrete", "inner_abc", "inner_protocol", "exception_method", + "shadowed_abc", "shadowed_exception", "shadowed_handler_log_message"): + self.assertIn(name, stub_names) + self.assertNotIn(name, context_names) + + def test_tool_reachability_and_heredoc_boundaries(self): + uncalled = run({ + "tools/runner.py": "def main(): return 0\\n", + "tools/README.md": "Use tools/runner.py.\\n", + "tests/test_runner.py": "def unused_helper():\\n import runner\\n", + }) + self.assertFalse(uncalled["tools"]["tools/runner.py"]["exercised"]) + dead = run({ + "tools/runner.py": "def main(): return 0\\n", + "tools/README.md": "Use tools/runner.py.\\n", + "tests/test_runner.py": "if False:\\n import runner\\n", + }) + self.assertFalse(dead["tools"]["tools/runner.py"]["exercised"]) + reachable = run({ + "tools/runner.py": "def main(): return 0\\n", + "tools/README.md": "Use tools/runner.py.\\n", + "tests/test_runner.py": ( + "import unittest\\n" + "class RunnerTests(unittest.TestCase):\\n" + " def test_reaches_helper(self): self.helper()\\n" + " def helper(self):\\n" + " import runner\\n" + ), + }) + self.assertTrue(reachable["tools"]["tools/runner.py"]["exercised"]) + workflow = ("jobs:\\n test:\\n steps:\\n - run: |\\n" + " cat <<'EOF'\\n" + " python tools/runner.py\\n" + " EOF\\n") + heredoc = run({ + "tools/runner.py": "def main(): return 0\\n", + "tools/README.md": "Use tools/runner.py.\\n", + ".github/workflows/ci.yml": workflow, + }) + self.assertFalse(heredoc["tools"]["tools/runner.py"]["exercised"]) + real_command = run({ + "tools/runner.py": "def main(): return 0\\n", + "tools/README.md": "Use tools/runner.py.\\n", + ".github/workflows/ci.yml": workflow + " python tools/runner.py\\n", + }) + self.assertTrue(real_command["tools"]["tools/runner.py"]["exercised"]) + + def test_mutation_inventory_is_sealed(self): + cases = g11m._build_cases() + self.assertEqual(len(cases), 74) + g11m._validate_mutation_inventory(cases) + with self.assertRaisesRegex(g11m.HarnessError, "mutation inventory"): + g11m._validate_mutation_inventory(cases[:-1]) + with self.assertRaisesRegex(g11m.HarnessError, "mutation inventory"): + g11m._validate_mutation_inventory(cases + [cases[0]]) + with self.assertRaisesRegex(g11m.HarnessError, "mutation inventory"): + g11m._validate_mutation_inventory(cases[1:] + cases[:1]) + + def test_mutation_child_environment_is_sanitized(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + tests_dir = root / "tests" + tests_dir.mkdir() + (tests_dir / "__init__.py").write_text("", encoding="utf-8") + (tests_dir / "test_probe.py").write_text( + "import os, unittest\\n" + "class Probe(unittest.TestCase):\\n" + " def test_environment(self):\\n" + " for key in ('PYTHONPATH', 'PYTHONHOME', 'PYTHONPYCACHEPREFIX'):\\n" + " self.assertNotIn(key, os.environ)\\n" + " self.assertNotIn('poison', unittest.__file__)\\n", + encoding="utf-8") + poison = root / "poison" + poison.mkdir() + (poison / "unittest.py").write_text("raise RuntimeError('poisoned unittest')\\n", encoding="utf-8") + env = os.environ.copy() + env["PYTHONPATH"] = str(poison) + env["PYTHONHOME"] = str(poison) + env["PYTHONPYCACHEPREFIX"] = str(root / "pycache") + outcome = g11m._run(str(root), "tests.test_probe", env=env) + self.assertEqual(outcome.get("status"), "OK") + self.assertEqual(outcome.get("ran"), 1) + + def test_in_scope_tracked_symlink_is_rejected(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + subprocess.run(["git", "init", "-q", str(root)], check=True, capture_output=True) + subprocess.run(["git", "-C", str(root), "config", "user.name", "G11 Test"], + check=True, capture_output=True) + subprocess.run(["git", "-C", str(root), "config", "user.email", "g11@example.invalid"], + check=True, capture_output=True) + target = root / "target.txt" + target.write_text("not a scanned source\\n", encoding="utf-8") + link = root / "tools" / "cbz" / "linked.py" + link.parent.mkdir(parents=True) + link.symlink_to(target) + subprocess.run(["git", "-C", str(root), "add", "tools/cbz/linked.py"], + check=True, capture_output=True) + subprocess.run(["git", "-C", str(root), "commit", "-qm", "symlink fixture"], + check=True, capture_output=True) + with self.assertRaisesRegex(ValueError, "in-scope non-regular tracked path"): + g11.read_tree(str(root), "HEAD") + + if __name__ == "__main__": unittest.main() \ No newline at end of file diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py index e9e9dce76..af218a98f 100644 --- a/tools/cbz/g11_mutation_selftest.py +++ b/tools/cbz/g11_mutation_selftest.py @@ -391,7 +391,7 @@ def run_selftest(root): results.append(dict(outcome, mutation=name, target=target, expected_test=test_id, expected_line=expected_line, killed=qualifies_kill(outcome, test_id, expected_line))) - baseline_ok = (baseline.get("parsed") and baseline.get("exit") == 0 and baseline.get("ran") == 33 + baseline_ok = (baseline.get("parsed") and baseline.get("exit") == 0 and baseline.get("ran") == 38 and baseline.get("failures") == 0 and baseline.get("errors") == 0 and baseline.get("skipped") == 0 and baseline.get("status") == "OK") killed = sum(r["killed"] for r in results) From d4d22d4afd0d484f09b8374d7e1ee20e3e678fb3 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:05:16 -0500 Subject: [PATCH 15/27] test(cbz): fix quoted python command fixture --- tests/test_g11_census.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_g11_census.py b/tests/test_g11_census.py index 701537333..739df6a6e 100644 --- a/tests/test_g11_census.py +++ b/tests/test_g11_census.py @@ -321,7 +321,7 @@ def test_comment_and_string_mentions_do_not_exercise_tools(self): python_cmd = run({ "tools/runner.py": "def main(): return 0\n", "tools/README.md": "Use tools/runner.py.\n", - "tests/test_runner.py": 'import subprocess\nsubprocess.run("python -c \\'tools/runner.py\\'", shell=True)\n', + "tests/test_runner.py": "import subprocess\nsubprocess.run(\"python -c 'tools/runner.py'\", shell=True)\n", }) self.assertFalse(python_cmd["tools"]["tools/runner.py"]["exercised"]) From 97584de646e5d7c79540626648165702280720b1 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:07:03 -0500 Subject: [PATCH 16/27] test(cbz): correct G11 residual regression fixtures --- tests/test_g11_census.py | 116 +++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/tests/test_g11_census.py b/tests/test_g11_census.py index 739df6a6e..805d36463 100644 --- a/tests/test_g11_census.py +++ b/tests/test_g11_census.py @@ -77,12 +77,12 @@ def test_real_bodies_are_not_stubs(self): self.assertEqual(stubs(text), []) def test_legitimate_contexts_are_reported_but_do_not_fail(self): - text = ("from abc import ABC, abstractmethod\\nfrom typing import Protocol, overload\\n" - "class Base(ABC):\\n def m(self):\\n ...\\n" - "class Shape(Protocol):\\n def area(self) -> float:\\n ...\\n" - "class Bad(ValueError):\\n def __init__(self):\\n pass\\n" - "class C:\\n @abstractmethod\\n def a(self):\\n pass\\n" - " @overload\\n def f(self, x: int) -> int:\\n ...\\n") + text = ("from abc import ABC, abstractmethod\nfrom typing import Protocol, overload\n" + "class Base(ABC):\n def m(self):\n ...\n" + "class Shape(Protocol):\n def area(self) -> float:\n ...\n" + "class Bad(ValueError):\n def __init__(self):\n pass\n" + "class C:\n @abstractmethod\n def a(self):\n pass\n" + " @overload\n def f(self, x: int) -> int:\n ...\n") report = run({"tools/good.py": text, **DOCUMENTED_TESTED}) contexts = sorted(f["detail"] for f in report["findings"] if f["class"] == "stub-context") self.assertEqual(contexts, ["a: pass; @abstractmethod", @@ -398,28 +398,28 @@ def test_mutation_kill_requires_exact_target_assertion(self): class ReviewResidualControls(unittest.TestCase): def test_stub_exemptions_are_immediate_and_scope_safe(self): text = ( - "from abc import ABC, abstractmethod\\n" - "from typing import Protocol\\n" - "from http.server import BaseHTTPRequestHandler\\n" - "class Abstract(ABC):\\n" - " @abstractmethod\\n" - " def required(self): ...\\n" - " def concrete(self): ...\\n" - " class Nested:\\n" - " def inner_abc(self): ...\\n" - "class Interface(Protocol):\\n" - " def declared(self): ...\\n" - " class Nested:\\n" - " def inner_protocol(self): ...\\n" - "class Failure(Exception):\\n" - " def exception_method(self): ...\\n" - "def build(ABC, Exception, BaseHTTPRequestHandler):\\n" - " class ShadowedABC(ABC):\\n" - " def shadowed_abc(self): ...\\n" - " class ShadowedError(Exception):\\n" - " def shadowed_exception(self): ...\\n" - " class ShadowedHandler(BaseHTTPRequestHandler):\\n" - " def shadowed_handler_log_message(self, *args): ...\\n" + "from abc import ABC, abstractmethod\n" + "from typing import Protocol\n" + "from http.server import BaseHTTPRequestHandler\n" + "class Abstract(ABC):\n" + " @abstractmethod\n" + " def required(self): ...\n" + " def concrete(self): ...\n" + " class Nested:\n" + " def inner_abc(self): ...\n" + "class Interface(Protocol):\n" + " def declared(self): ...\n" + " class Nested:\n" + " def inner_protocol(self): ...\n" + "class Failure(Exception):\n" + " def exception_method(self): ...\n" + "def build(ABC, Exception, BaseHTTPRequestHandler):\n" + " class ShadowedABC(ABC):\n" + " def shadowed_abc(self): ...\n" + " class ShadowedError(Exception):\n" + " def shadowed_exception(self): ...\n" + " class ShadowedHandler(BaseHTTPRequestHandler):\n" + " def shadowed_handler_log_message(self, *args): ...\n" ) report = run({"tools/good.py": text, **DOCUMENTED_TESTED}) stub_names = {f["detail"].split(":")[0] for f in report["findings"] if f["class"] == "stub"} @@ -433,43 +433,43 @@ def test_stub_exemptions_are_immediate_and_scope_safe(self): def test_tool_reachability_and_heredoc_boundaries(self): uncalled = run({ - "tools/runner.py": "def main(): return 0\\n", - "tools/README.md": "Use tools/runner.py.\\n", - "tests/test_runner.py": "def unused_helper():\\n import runner\\n", + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + "tests/test_runner.py": "def unused_helper():\n import runner\n", }) self.assertFalse(uncalled["tools"]["tools/runner.py"]["exercised"]) dead = run({ - "tools/runner.py": "def main(): return 0\\n", - "tools/README.md": "Use tools/runner.py.\\n", - "tests/test_runner.py": "if False:\\n import runner\\n", + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + "tests/test_runner.py": "if False:\n import runner\n", }) self.assertFalse(dead["tools"]["tools/runner.py"]["exercised"]) reachable = run({ - "tools/runner.py": "def main(): return 0\\n", - "tools/README.md": "Use tools/runner.py.\\n", + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", "tests/test_runner.py": ( - "import unittest\\n" - "class RunnerTests(unittest.TestCase):\\n" - " def test_reaches_helper(self): self.helper()\\n" - " def helper(self):\\n" - " import runner\\n" + "import unittest\n" + "class RunnerTests(unittest.TestCase):\n" + " def test_reaches_helper(self): self.helper()\n" + " def helper(self):\n" + " import runner\n" ), }) self.assertTrue(reachable["tools"]["tools/runner.py"]["exercised"]) - workflow = ("jobs:\\n test:\\n steps:\\n - run: |\\n" - " cat <<'EOF'\\n" - " python tools/runner.py\\n" - " EOF\\n") + workflow = ("jobs:\n test:\n steps:\n - run: |\n" + " cat <<'EOF'\n" + " python tools/runner.py\n" + " EOF\n") heredoc = run({ - "tools/runner.py": "def main(): return 0\\n", - "tools/README.md": "Use tools/runner.py.\\n", + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", ".github/workflows/ci.yml": workflow, }) self.assertFalse(heredoc["tools"]["tools/runner.py"]["exercised"]) real_command = run({ - "tools/runner.py": "def main(): return 0\\n", - "tools/README.md": "Use tools/runner.py.\\n", - ".github/workflows/ci.yml": workflow + " python tools/runner.py\\n", + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + ".github/workflows/ci.yml": workflow + " python tools/runner.py\n", }) self.assertTrue(real_command["tools"]["tools/runner.py"]["exercised"]) @@ -491,16 +491,16 @@ def test_mutation_child_environment_is_sanitized(self): tests_dir.mkdir() (tests_dir / "__init__.py").write_text("", encoding="utf-8") (tests_dir / "test_probe.py").write_text( - "import os, unittest\\n" - "class Probe(unittest.TestCase):\\n" - " def test_environment(self):\\n" - " for key in ('PYTHONPATH', 'PYTHONHOME', 'PYTHONPYCACHEPREFIX'):\\n" - " self.assertNotIn(key, os.environ)\\n" - " self.assertNotIn('poison', unittest.__file__)\\n", + "import os, unittest\n" + "class Probe(unittest.TestCase):\n" + " def test_environment(self):\n" + " for key in ('PYTHONPATH', 'PYTHONHOME', 'PYTHONPYCACHEPREFIX'):\n" + " self.assertNotIn(key, os.environ)\n" + " self.assertNotIn('poison', unittest.__file__)\n", encoding="utf-8") poison = root / "poison" poison.mkdir() - (poison / "unittest.py").write_text("raise RuntimeError('poisoned unittest')\\n", encoding="utf-8") + (poison / "unittest.py").write_text("raise RuntimeError('poisoned unittest')\n", encoding="utf-8") env = os.environ.copy() env["PYTHONPATH"] = str(poison) env["PYTHONHOME"] = str(poison) @@ -518,7 +518,7 @@ def test_in_scope_tracked_symlink_is_rejected(self): subprocess.run(["git", "-C", str(root), "config", "user.email", "g11@example.invalid"], check=True, capture_output=True) target = root / "target.txt" - target.write_text("not a scanned source\\n", encoding="utf-8") + target.write_text("not a scanned source\n", encoding="utf-8") link = root / "tools" / "cbz" / "linked.py" link.parent.mkdir(parents=True) link.symlink_to(target) From 609783fc041771426e280d6662be7fcc3eee8755 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:22:13 -0500 Subject: [PATCH 17/27] fix(cbz): close G11 auditor soundness residuals --- .github/workflows/cbz-g11-auditor.yml | 8 +- docs/cbz/G11_AUDITOR.md | 10 +- tests/test_g11_census.py | 44 ++--- tools/cbz/g11_census.py | 251 ++++++++++++++++++++------ tools/cbz/g11_mutation_selftest.py | 245 +++++++++++++++++++------ 5 files changed, 415 insertions(+), 143 deletions(-) diff --git a/.github/workflows/cbz-g11-auditor.yml b/.github/workflows/cbz-g11-auditor.yml index 0c0dbce27..0d3fe0c64 100644 --- a/.github/workflows/cbz-g11-auditor.yml +++ b/.github/workflows/cbz-g11-auditor.yml @@ -19,10 +19,10 @@ jobs: with: python-version: '3.11' - name: Run census controls normally - run: python3 -B -S -m unittest tests.test_g11_census + run: env -u PYTHONPATH -u PYTHONHOME -u PYTHONPYCACHEPREFIX -u PYTHONSTARTUP -u PYTHONUSERBASE python3 -B -S -m unittest tests.test_g11_census - name: Run census controls with optimization - run: python3 -B -S -O -m unittest tests.test_g11_census + run: env -u PYTHONPATH -u PYTHONHOME -u PYTHONPYCACHEPREFIX -u PYTHONSTARTUP -u PYTHONUSERBASE python3 -B -S -O -m unittest tests.test_g11_census - name: Run mutation self-test normally - run: python3 -B -S tools/cbz/g11_mutation_selftest.py + run: env -u PYTHONPATH -u PYTHONHOME -u PYTHONPYCACHEPREFIX -u PYTHONSTARTUP -u PYTHONUSERBASE python3 -B -S tools/cbz/g11_mutation_selftest.py - name: Run mutation self-test with optimization - run: python3 -B -S -O tools/cbz/g11_mutation_selftest.py + run: env -u PYTHONPATH -u PYTHONHOME -u PYTHONPYCACHEPREFIX -u PYTHONSTARTUP -u PYTHONUSERBASE python3 -B -S -O tools/cbz/g11_mutation_selftest.py diff --git a/docs/cbz/G11_AUDITOR.md b/docs/cbz/G11_AUDITOR.md index 763f78eb6..335e6f17f 100644 --- a/docs/cbz/G11_AUDITOR.md +++ b/docs/cbz/G11_AUDITOR.md @@ -2,7 +2,7 @@ ## Purpose -`tools/cbz/g11_census.py` reads tracked blob bytes from one Git revision and reports review triggers for the owner-defined G11 tooling-hygiene criterion: Python stubs, code-comment markers, no-op workflow run steps, and tool documentation/exercise references. It is a triage instrument, not a proof, a Layer A certificate, or a federation-wide G11 admission. +`tools/cbz/g11_census.py` reads tracked blob bytes from one Git revision and reports review triggers for the owner-defined G11 tooling-hygiene criterion: Python stubs, code-comment markers, no-op workflow run steps, and tool documentation/static-reference checks. It is a triage instrument, not a proof, a Layer A certificate, or a federation-wide G11 admission. ## Commands @@ -16,14 +16,14 @@ python3 -B -S -O tools/cbz/g11_mutation_selftest.py python3 -B -S tools/cbz/g11_census.py --rev --json census.json ``` -The census exits 0 for no detected production finding, 1 for a detected production finding, and 2 when the requested Git revision cannot be read. A zero exit is only the result of this implementation over its stated scanned paths and rules. +The census exits 0 for no detected production finding, 1 for a detected production finding, and 2 when the requested Git revision cannot be read. In-scope tracked symlinks and other non-regular scanned entries fail closed with exit 2. A zero exit is only the result of this implementation over its stated scanned paths and rules. ## Soundness boundaries -- Stub exemptions are bound to imported `abc.abstractmethod`, `typing.overload`, `typing_extensions.overload`, `abc.ABC`, `typing.Protocol`, `typing_extensions.Protocol`, actual built-in exception classes, and the named `http.server` request-handler bases. Name suffixes, locally defined look-alikes, and enclosing function/class bindings that shadow an imported decorator do not grant exemptions. -- Python test/tool exercise evidence uses parsed imports and recognized subprocess invocations, not arbitrary comments or string mentions. Python and shell `-c` payloads are deliberately not credited from path mentions alone. JavaScript test imports are read from static import declarations after comment and template-literal masking; dynamic imports are outside this recognizer. Workflow exercise evidence is read from run commands, not YAML comments. +- Stub exemptions are bound to imported `abc.abstractmethod`, `typing.overload`, `typing_extensions.overload`, immediate `typing.Protocol`/`typing_extensions.Protocol` interface methods, and the named `http.server` request-handler `log_message` override. ABC inheritance alone and exception-class inheritance do not exempt a method. Nested outer-class context, name suffixes, local look-alikes, and lexical bindings shadowing an imported decorator/base do not grant exemptions. +- Python reference evidence is limited to module-level statements and `unittest.TestCase` methods, following only locally reachable helper calls and literal branches; uncalled helpers, `if False`, runtime-dependent branches, and loop bodies are not credited. Workflow references ignore here-document bodies and are read from run commands, not YAML comments. Python/shell `-c` payloads and JavaScript template-literal text are not credited. These are static references only, never proof of runtime execution, and cannot satisfy a runtime-exercise requirement. - YAML comments are scanned for marker words. Empty and comment-only `run` blocks are reported as no-op steps. -- The mutation self-test preserves the original mutation inventory and admits a kill only when the clean full 33-test baseline succeeds and the exact designated module/class/method reports `FAILED` with one AssertionError at its exact designated assertion line, zero errors, and zero skips. The harness propagates the parent's optimized mode to child tests and mutates its own parser and each attribution guard. +- The mutation self-test preserves the original mutation inventory and admits a kill only when the clean full 38-test baseline succeeds and the exact designated module/class/method reports `FAILED` with one AssertionError at its exact designated assertion line, zero errors, and zero skips. The harness propagates the parent's optimized mode to child tests and mutates its own parser and each attribution guard. ## Limits diff --git a/tests/test_g11_census.py b/tests/test_g11_census.py index 805d36463..23a479702 100644 --- a/tests/test_g11_census.py +++ b/tests/test_g11_census.py @@ -182,13 +182,13 @@ def test_documented_and_exercised_by_path_import_and_basename(self): for path in ("tools/a.py", "tools/b.py", "tools/c.mjs", "tools/d.py"): with self.subTest(tool=path): self.assertTrue(report[path]["documented"]) - self.assertTrue(report[path]["exercised"]) + self.assertTrue(report[path]["referenced"]) def test_undocumented_and_unexercised_tools_fail(self): result = run({"tools/lonely.py": "x = 1\n", "tools/README.md": "nothing here\n"}) self.assertEqual(result["failing_tools"], ["tools/lonely.py"]) self.assertFalse(result["tools"]["tools/lonely.py"]["documented"]) - self.assertFalse(result["tools"]["tools/lonely.py"]["exercised"]) + self.assertFalse(result["tools"]["tools/lonely.py"]["referenced"]) self.assertEqual(result["status"], "FAIL") def test_exercised_through_an_exercised_tool(self): @@ -196,24 +196,24 @@ def test_exercised_through_an_exercised_tool(self): "tools/README.md": "tools/top.py tools/helper.py tools/orphan.py\n", "tests/test_top.py": "import top\n"} report = self.tools(tree) - self.assertEqual(report["tools/helper.py"]["exercised_by"], ["via tools/top.py"]) - self.assertFalse(report["tools/orphan.py"]["exercised"]) + self.assertEqual(report["tools/helper.py"]["referenced_by"], ["via tools/top.py"]) + self.assertFalse(report["tools/orphan.py"]["referenced"]) def test_ambiguous_basename_needs_the_full_path(self): tree = {"tools/run.py": "x = 1\n", "docs/USE.md": "call run.py\n", "tests/test_x.py": "import run\n"} ambiguous = self.tools(tree, sorted(tree) + ["experiments/run.py"]) self.assertFalse(ambiguous["tools/run.py"]["documented"]) - self.assertFalse(ambiguous["tools/run.py"]["exercised"]) + self.assertFalse(ambiguous["tools/run.py"]["referenced"]) unique = self.tools(tree) self.assertTrue(unique["tools/run.py"]["documented"]) - self.assertTrue(unique["tools/run.py"]["exercised"]) + self.assertTrue(unique["tools/run.py"]["referenced"]) def test_full_path_counts_even_when_basename_is_ambiguous(self): tree = {"tools/run.py": "x = 1\n", "docs/USE.md": "call tools/run.py\n", ".github/workflows/ci.yml": " - run: python tools/run.py\n"} report = self.tools(tree, sorted(tree) + ["experiments/run.py"]) self.assertTrue(report["tools/run.py"]["documented"]) - self.assertTrue(report["tools/run.py"]["exercised"]) + self.assertTrue(report["tools/run.py"]["referenced"]) def test_module_docstring_is_recorded(self): report = self.tools({"tools/a.py": '"""Does a."""\n', "tools/b.py": "x = 1\n"}) @@ -315,7 +315,7 @@ def test_comment_and_string_mentions_do_not_exercise_tools(self): "tests/test_runner.py": "# tools/runner.py\nDOC = 'tools/runner.py'\nprint('tools/runner.py')\n", } report = run(tree) - self.assertFalse(report["tools"]["tools/runner.py"]["exercised"]) + self.assertFalse(report["tools"]["tools/runner.py"]["referenced"]) self.assertEqual(report["failing_tools"], ["tools/runner.py"]) python_cmd = run({ @@ -323,7 +323,7 @@ def test_comment_and_string_mentions_do_not_exercise_tools(self): "tools/README.md": "Use tools/runner.py.\n", "tests/test_runner.py": "import subprocess\nsubprocess.run(\"python -c 'tools/runner.py'\", shell=True)\n", }) - self.assertFalse(python_cmd["tools"]["tools/runner.py"]["exercised"]) + self.assertFalse(python_cmd["tools"]["tools/runner.py"]["referenced"]) workflow_text = "jobs:\n test:\n steps:\n - run: \"sh -c 'echo tools/runner.py'\"\n" shell_cmd = run({ @@ -332,14 +332,14 @@ def test_comment_and_string_mentions_do_not_exercise_tools(self): ".github/workflows/ci.yml": workflow_text, }) self.assertEqual(g11._workflow_commands(workflow_text)[0][1], ["sh -c 'echo tools/runner.py'"]) - self.assertFalse(shell_cmd["tools"]["tools/runner.py"]["exercised"]) + self.assertFalse(shell_cmd["tools"]["tools/runner.py"]["referenced"]) js_cmd = run({ "tools/runner.mjs": "export const run = () => 1;\n", "tools/README.md": "Use tools/runner.mjs.\n", "tests/runner.test.mjs": "const sample = `\nimport '../tools/runner.mjs';\n`;\n", }) - self.assertFalse(js_cmd["tools"]["tools/runner.mjs"]["exercised"]) + self.assertFalse(js_cmd["tools"]["tools/runner.mjs"]["referenced"]) def test_empty_and_comment_only_run_blocks_fail(self): text = ("jobs:\n a:\n steps:\n" @@ -426,6 +426,8 @@ def test_stub_exemptions_are_immediate_and_scope_safe(self): context_names = {f["detail"].split(":")[0] for f in report["findings"] if f["class"] == "stub-context"} self.assertIn("required", context_names) self.assertIn("declared", context_names) + self.assertIn("inner_abc", stub_names) + self.assertIn("shadowed_abc", stub_names) for name in ("concrete", "inner_abc", "inner_protocol", "exception_method", "shadowed_abc", "shadowed_exception", "shadowed_handler_log_message"): self.assertIn(name, stub_names) @@ -437,13 +439,13 @@ def test_tool_reachability_and_heredoc_boundaries(self): "tools/README.md": "Use tools/runner.py.\n", "tests/test_runner.py": "def unused_helper():\n import runner\n", }) - self.assertFalse(uncalled["tools"]["tools/runner.py"]["exercised"]) + self.assertFalse(uncalled["tools"]["tools/runner.py"]["referenced"]) dead = run({ "tools/runner.py": "def main(): return 0\n", "tools/README.md": "Use tools/runner.py.\n", "tests/test_runner.py": "if False:\n import runner\n", }) - self.assertFalse(dead["tools"]["tools/runner.py"]["exercised"]) + self.assertFalse(dead["tools"]["tools/runner.py"]["referenced"]) reachable = run({ "tools/runner.py": "def main(): return 0\n", "tools/README.md": "Use tools/runner.py.\n", @@ -455,7 +457,7 @@ def test_tool_reachability_and_heredoc_boundaries(self): " import runner\n" ), }) - self.assertTrue(reachable["tools"]["tools/runner.py"]["exercised"]) + self.assertTrue(reachable["tools"]["tools/runner.py"]["referenced"]) workflow = ("jobs:\n test:\n steps:\n - run: |\n" " cat <<'EOF'\n" " python tools/runner.py\n" @@ -465,24 +467,22 @@ def test_tool_reachability_and_heredoc_boundaries(self): "tools/README.md": "Use tools/runner.py.\n", ".github/workflows/ci.yml": workflow, }) - self.assertFalse(heredoc["tools"]["tools/runner.py"]["exercised"]) + self.assertFalse(heredoc["tools"]["tools/runner.py"]["referenced"]) real_command = run({ "tools/runner.py": "def main(): return 0\n", "tools/README.md": "Use tools/runner.py.\n", ".github/workflows/ci.yml": workflow + " python tools/runner.py\n", }) - self.assertTrue(real_command["tools"]["tools/runner.py"]["exercised"]) + self.assertTrue(real_command["tools"]["tools/runner.py"]["referenced"]) def test_mutation_inventory_is_sealed(self): cases = g11m._build_cases() self.assertEqual(len(cases), 74) g11m._validate_mutation_inventory(cases) - with self.assertRaisesRegex(g11m.HarnessError, "mutation inventory"): - g11m._validate_mutation_inventory(cases[:-1]) - with self.assertRaisesRegex(g11m.HarnessError, "mutation inventory"): - g11m._validate_mutation_inventory(cases + [cases[0]]) - with self.assertRaisesRegex(g11m.HarnessError, "mutation inventory"): - g11m._validate_mutation_inventory(cases[1:] + cases[:1]) + altered_cases = (cases[:-1], cases + [cases[0]], cases[1:] + cases[:1]) + for altered in altered_cases: + with self.assertRaisesRegex(g11m.HarnessError, "mutation inventory"): + g11m._validate_mutation_inventory(altered) def test_mutation_child_environment_is_sanitized(self): with tempfile.TemporaryDirectory() as temporary: diff --git a/tools/cbz/g11_census.py b/tools/cbz/g11_census.py index 80a8496f0..bb0e62587 100644 --- a/tools/cbz/g11_census.py +++ b/tools/cbz/g11_census.py @@ -5,17 +5,16 @@ stub a Python function whose body, after any docstring, is only `pass`, `...` or `raise NotImplementedError`, or is the docstring alone. - A stub under @abstractmethod or @overload, or in a Protocol, ABC or - exception class, is reported with that context and does not fail. + @abstractmethod/@overload and immediate Protocol methods are + contextual; ABC or exception inheritance alone is not an exemption. marker TODO, FIXME, XXX, TBD or STUB (upper case) or "not implemented" in a code comment. In Python the comment is found by tokenizing; in .mjs/.js/.cjs/.sh a comment is found by a quote-parity heuristic. The same words in string data are recorded as mentions only. noop a workflow step whose `run:` does nothing: `true`, `:` or `exit 0`. tools every executable source under tools/ (not tests, not __init__.py), - with the tracked Markdown files that name it and the evidence that - something exercises it: a test or workflow naming or importing it, - or an import from a tool that is itself exercised. + with tracked Markdown documentation and statically reachable test + or workflow references. References are not proof of runtime execution. Every path has an area. Only `tooling` (tools/, .github/) and `site` (docs/site/ outside vendor/) are production; findings elsewhere are reported @@ -183,31 +182,39 @@ def _resolve_import(node, imports): return tuple(resolved.rsplit(".", 1)) -def _trusted_decorator(node, imports, scope_shadowed=()): +def _resolve_scoped_import(node, imports, scope_shadowed=()): target = node - while isinstance(target, ast.Call): - target = target.func + while isinstance(target, (ast.Call, ast.Subscript)): + target = target.func if isinstance(target, ast.Call) else target.value dotted = _dotted_name(target) binding = dotted.split(".", 1)[0] if dotted else "" if binding and binding in scope_shadowed: - return False - resolved = _resolve_import(target, imports) + return None + return _resolve_import(target, imports) + + +def _trusted_decorator(node, imports, scope_shadowed=()): + resolved = _resolve_scoped_import(node, imports, scope_shadowed) return resolved in {("abc", "abstractmethod"), ("typing", "overload"), ("typing_extensions", "overload")} -def _trusted_abstract_base(node, imports): - resolved = _resolve_import(node, imports) - return resolved in {("abc", "ABC"), ("typing", "Protocol"), - ("typing_extensions", "Protocol")} +def _trusted_abstract_base(node, imports, scope_shadowed=()): + resolved = _resolve_scoped_import(node, imports, scope_shadowed) + if resolved == ("abc", "ABC"): + return "ABC" + if resolved in {("typing", "Protocol"), ("typing_extensions", "Protocol")}: + return "Protocol" + return None -def _builtin_exception_base(node, imports): - resolved = _resolve_import(node, imports) +def _builtin_exception_base(node, imports, scope_shadowed=()): + resolved = _resolve_scoped_import(node, imports, scope_shadowed) name = None if resolved and resolved[0] == "builtins": name = resolved[1] - elif isinstance(node, ast.Name) and node.id in builtins.__dict__ and node.id not in imports["shadowed"]: + elif (isinstance(node, ast.Name) and node.id in builtins.__dict__ + and node.id not in imports["shadowed"] and node.id not in scope_shadowed): name = node.id if not name: return False @@ -215,8 +222,8 @@ def _builtin_exception_base(node, imports): return isinstance(candidate, type) and issubclass(candidate, BaseException) -def _http_request_handler_base(node, imports): - return _resolve_import(node, imports) in { +def _http_request_handler_base(node, imports, scope_shadowed=()): + return _resolve_scoped_import(node, imports, scope_shadowed) in { ("http.server", "BaseHTTPRequestHandler"), ("http.server", "SimpleHTTPRequestHandler"), ("http.server", "CGIHTTPRequestHandler"), @@ -250,14 +257,18 @@ def _legit_context(func, classes, imports, scope_shadowed=()): for dec in func.decorator_list: if _trusted_decorator(dec, imports, scope_shadowed): return "@" + _decorator_name(dec) - for cls in reversed(classes): + # Only the immediate class can supply method context. + for cls in classes[-1:]: for base in cls.bases: - name = _base_name(base) - if _trusted_abstract_base(base, imports): - return "method of a %s class" % name - if _builtin_exception_base(base, imports): - return "method of an exception class" - if _http_request_handler_base(base, imports) and func.name == "log_message": + base_kind = _trusted_abstract_base(base, imports, scope_shadowed) + if base_kind == "Protocol": + return "method of a Protocol class" + if base_kind == "ABC": + continue + if _builtin_exception_base(base, imports, scope_shadowed): + continue + if (_http_request_handler_base(base, imports, scope_shadowed) + and func.name == "log_message"): return "log_message override that silences http.server request logging" return None @@ -487,13 +498,119 @@ def _tool_for_token(token, tools): return None -def python_tool_references(text, tools, unique): +def _literal_bool(node): + try: + return bool(ast.literal_eval(node)) + except (ValueError, TypeError): + return None + + +def _is_main_guard(node): + return (isinstance(node, ast.Compare) and isinstance(node.left, ast.Name) + and node.left.id == "__name__" and len(node.ops) == 1 + and isinstance(node.ops[0], ast.Eq) and len(node.comparators) == 1 + and isinstance(node.comparators[0], ast.Constant) + and node.comparators[0].value == "__main__") + + +def _reachable_python_nodes(tree, mode): + functions, methods = {}, {} + + def register(body, owner=None): + for statement in body: + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + if owner is None: + functions.setdefault(statement.name, statement) + else: + methods.setdefault((owner, statement.name), statement) + elif isinstance(statement, ast.ClassDef): + register(statement.body, statement.name) + + register(tree.body) + test_classes = set() + if mode == "tests": + for statement in tree.body: + if (isinstance(statement, ast.ClassDef) + and any(_base_name(base) == "TestCase" for base in statement.bases)): + test_classes.add(statement.name) + + nodes, queue, visited = [], [], set() + + def visit(node, owner=None): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + # A definition is not a call: only decorators/defaults execute here. + for decorator in node.decorator_list: + visit(decorator, owner) + for default in node.args.defaults: + visit(default, owner) + for default in node.args.kw_defaults: + if default is not None: + visit(default, owner) + if mode == "tests" and owner in test_classes and node.name.startswith("test_"): + queue.append((node, owner)) + return + if isinstance(node, ast.ClassDef): + for decorator in node.decorator_list: + visit(decorator, owner) + for base in node.bases: + visit(base, owner) + for keyword in node.keywords: + visit(keyword.value, owner) + for statement in node.body: + visit(statement, node.name) + return + if isinstance(node, ast.If): + visit(node.test, owner) + if _is_main_guard(node.test): + branch = node.body if mode == "script" else node.orelse + else: + truth = _literal_bool(node.test) + if truth is None: + return + branch = node.body if truth else node.orelse + for statement in branch: + visit(statement, owner) + return + if isinstance(node, (ast.For, ast.AsyncFor, ast.While)): + if isinstance(node, (ast.For, ast.AsyncFor)): + visit(node.iter, owner) + else: + visit(node.test, owner) + return + + nodes.append(node) + if isinstance(node, ast.Call): + target = node.func + if isinstance(target, ast.Name) and target.id in functions: + queue.append((functions[target.id], None)) + elif (isinstance(target, ast.Attribute) and isinstance(target.value, ast.Name) + and target.value.id in {"self", "cls"} and owner): + method = methods.get((owner, target.attr)) + if method: + queue.append((method, owner)) + for child in ast.iter_child_nodes(node): + visit(child, owner) + + for statement in tree.body: + visit(statement) + while queue: + function, owner = queue.pop() + identity = (id(function), owner) + if identity in visited: + continue + visited.add(identity) + for statement in function.body: + visit(statement, owner) + return nodes + + +def python_tool_references(text, tools, unique, mode="script"): references = set() try: tree = ast.parse(text) except (SyntaxError, ValueError): return references - for node in ast.walk(tree): + for node in _reachable_python_nodes(tree, mode): if isinstance(node, ast.Import): for alias in node.names: path = _tool_for_module(alias.name, tools, unique) @@ -526,7 +643,6 @@ def python_tool_references(text, tools, unique): else: tokens = [] if tokens and tokens[0] in {"python", "python3", "python3.11", "bash", "sh"}: - # Inline -c payloads are code strings, not proof that a path was executed. if "-c" in tokens[1:]: continue for token in tokens[1:]: @@ -624,36 +740,52 @@ def js_tool_references(text, source_path, tools): return references +def _heredoc_opener(command): + match = re.search(r"<<(?P-?)(?:'(?P[^']+)'|\"(?P[^\"]+)\"|(?P[A-Za-z_][A-Za-z0-9_]*))", + command) + if not match: + return None + delimiter = match.group("single") or match.group("double") or match.group("bare") + return delimiter, match.group("strip") == "-" + + def workflow_tool_references(text, tools): references = set() for _number, commands in _workflow_commands(text): + heredoc = None for command in commands: + if heredoc: + delimiter, strip_tabs = heredoc + candidate = command.lstrip("\t") if strip_tabs else command + if candidate.strip() == delimiter: + heredoc = None + continue try: tokens = shlex.split(command) except ValueError: - continue - if not tokens: - continue - if tokens[0] in {"python", "python3", "python3.11", "bash", "sh"}: - # Do not treat a path mentioned inside an inline -c program as an invocation. - if "-c" in tokens[1:]: - continue - for index, token in enumerate(tokens[1:], 1): - if token == "-m" and index + 1 < len(tokens): - module_path = tokens[index + 1].replace(".", "/") + ".py" - if module_path in tools: - references.add(module_path) - break - if token.startswith("-"): - continue - path = _tool_for_token(token, tools) + tokens = [] + if tokens: + if tokens[0] in {"python", "python3", "python3.11", "bash", "sh"}: + if "-c" not in tokens[1:]: + for index, token in enumerate(tokens[1:], 1): + if token == "-m" and index + 1 < len(tokens): + module_path = tokens[index + 1].replace(".", "/") + ".py" + if module_path in tools: + references.add(module_path) + break + if token.startswith("-"): + continue + path = _tool_for_token(token, tools) + if path: + references.add(path) + break + else: + path = _tool_for_token(tokens[0], tools) if path: references.add(path) - break - else: - path = _tool_for_token(tokens[0], tools) - if path: - references.add(path) + opener = _heredoc_opener(command) + if opener: + heredoc = opener return references @@ -674,10 +806,10 @@ def tool_report(files, texts, all_paths): (p.startswith(".github/workflows/") and p.endswith((".yml", ".yaml")))] direct, tool_dependencies = {p: [] for p in tools}, {} for path in tools: - tool_dependencies[path] = python_tool_references(texts.get(path, ""), tools, unique_modules) + tool_dependencies[path] = python_tool_references(texts.get(path, ""), tools, unique_modules, mode="script") for exerciser in exercisers: if exerciser.endswith(".py"): - refs = python_tool_references(texts[exerciser], tools, unique_modules) + refs = python_tool_references(texts[exerciser], tools, unique_modules, mode="tests") elif exerciser.endswith((".mjs", ".js", ".cjs")): refs = js_tool_references(texts[exerciser], exerciser, tools) else: @@ -695,18 +827,18 @@ def tool_report(files, texts, all_paths): except (SyntaxError, ValueError): docstring = False report[path] = {"documented_by": documented[:5], "documented": bool(documented), - "exercised_by": direct_paths[:5], "exercised": bool(direct_paths), + "referenced_by": direct_paths[:5], "referenced": bool(direct_paths), "module_docstring": docstring} changed = True while changed: changed = False for path in tools: - if report[path]["exercised"] or path not in texts: + if report[path]["referenced"] or path not in texts: continue for other in tools: - if other != path and report[other]["exercised"] and other in texts and path in tool_dependencies.get(other, set()): - report[path]["exercised"] = True - report[path]["exercised_by"] = ["via " + other] + if other != path and report[other]["referenced"] and other in texts and path in tool_dependencies.get(other, set()): + report[path]["referenced"] = True + report[path]["referenced_by"] = ["via " + other] changed = True break return report @@ -753,7 +885,7 @@ def census(files, all_paths=None): tools = tool_report(files, texts, sorted(all_paths if all_paths is not None else files)) failing = [f for f in findings if f["area"] in PRODUCTION and f["class"] in {"stub", "marker", "noop", "undecodable", "unparseable"}] - failing_tools = sorted(p for p, t in tools.items() if not (t["documented"] and t["exercised"])) + failing_tools = sorted(p for p, t in tools.items() if not (t["documented"] and t["referenced"])) return {"findings": findings, "tools": tools, "failing": failing, "failing_tools": failing_tools, "status": "FAIL" if failing or failing_tools else "PASS"} @@ -768,6 +900,9 @@ def read_tree(root, rev): meta, path = item.split(b"\t", 1) mode, kind, sha = meta.split() path = path.decode("utf-8", "surrogateescape") + if path.endswith(SCANNED) and (kind != b"blob" or mode == b"120000"): + raise ValueError("in-scope non-regular tracked path %s (mode %s, type %s)" % + (path, mode.decode(), kind.decode())) if kind == b"blob" and mode != b"120000": if path.endswith(SCANNED): entries.append((path, sha.decode())) diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py index af218a98f..02f5f9e11 100644 --- a/tools/cbz/g11_mutation_selftest.py +++ b/tools/cbz/g11_mutation_selftest.py @@ -45,11 +45,13 @@ " if not rest:\n return None\n"), ("docstring not stripped", " rest = rest[1:]\n", " rest = rest[0:]\n"), ("decorator context ignored", " if _trusted_decorator(dec, imports, scope_shadowed):", " if False:"), - ("Protocol/ABC context ignored", " if _trusted_abstract_base(base, imports):", " if False:"), - ("exception context ignored", " if _builtin_exception_base(base, imports):", " if False:"), - ("log_message context ignored", ' if _http_request_handler_base(base, imports) and func.name == "log_message":', " if False:"), - ("log_message context too broad", ' if _http_request_handler_base(base, imports) and func.name == "log_message":', - ' if _http_request_handler_base(base, imports):'), + ("Protocol/ABC context ignored", ' if base_kind == "Protocol":', " if False:"), + ("exception context ignored", + " if _builtin_exception_base(base, imports, scope_shadowed):\n continue\n", + " if _builtin_exception_base(base, imports, scope_shadowed):\n return \"method of an exception class\"\n"), + ("log_message context ignored", ' if _http_request_handler_base(base, imports, scope_shadowed) and func.name == "log_message":', " if False:"), + ("log_message context too broad", ' if _http_request_handler_base(base, imports, scope_shadowed) and func.name == "log_message":', + ' if _http_request_handler_base(base, imports, scope_shadowed):'), ("module docstring not recorded", " docstring = bool(ast.get_docstring(ast.parse(files[path])))", " docstring = True"), ("class bodies not visited", " visit(child, classes + [child], scope_shadowed | _scope_bindings(child))", " pass"), @@ -91,8 +93,8 @@ ("from-import list not recognised", " elif isinstance(node, ast.ImportFrom) and node.level == 0:", " elif False:"), ("basename not recognised", ' r"|\\b" + re.escape(path.rsplit("/", 1)[-1]) + r"\\b")', ' r"|\\bNOMATCH")'), - ("transitive exercise off", ' if other != path and report[other]["exercised"] and other in texts and path in tool_dependencies.get(other, set()):', - ' if False and other != path and report[other]["exercised"] and other in texts and path in tool_dependencies.get(other, set()):'), + ("transitive exercise off", ' if other != path and report[other]["referenced"] and other in texts and path in tool_dependencies.get(other, set()):', + ' if False and other != path and report[other]["referenced"] and other in texts and path in tool_dependencies.get(other, set()):'), ("site not production", 'PRODUCTION = {"tooling", "site"}', 'PRODUCTION = {"tooling"}'), ("research counted as production", 'PRODUCTION = {"tooling", "site"}', 'PRODUCTION = {"tooling", "site", "research"}'), ("vendor counted as site", ' if path.startswith("docs/site/vendor/"):\n return "vendor"', @@ -132,7 +134,7 @@ class HarnessError(Exception): "docstring not stripped": ("tests.test_g11_census.PythonStubs.test_docstring_then_stub_body", "Doc."), "decorator context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), "Protocol/ABC context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), - "exception context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), + "exception context ignored": ("tests.test_g11_census.ReviewResidualControls.test_stub_exemptions_are_immediate_and_scope_safe", 'self.assertIn("exception_method", stub_names)'), "log_message context ignored": ("tests.test_g11_census.PythonStubs.test_request_handler_log_message_override_is_legitimate", "self.assertEqual(found,"), "log_message context too broad": ("tests.test_g11_census.PythonStubs.test_request_handler_log_message_override_is_legitimate", "self.assertEqual(found,"), "module docstring not recorded": ("tests.test_g11_census.ToolChecks.test_module_docstring_is_recorded", "self.assertEqual((report["), @@ -156,16 +158,16 @@ class HarnessError(Exception): "__init__ counted as tool": ("tests.test_g11_census.ToolChecks.test_tests_and_init_files_are_not_tools", "self.assertEqual(sorted(report),"), "basename ambiguity ignored": ("tests.test_g11_census.ToolChecks.test_ambiguous_basename_needs_the_full_path", 'self.assertFalse(report["tools/run.py"]["documented"])'), "path reference not recognised": ("tests.test_g11_census.ToolChecks.test_full_path_counts_even_when_basename_is_ambiguous", 'self.assertTrue(report["tools/run.py"]["documented"])'), - "import not recognised": ("tests.test_g11_census.ToolChecks.test_documented_and_exercised_by_path_import_and_basename", 'self.assertTrue(report[path]["exercised"])'), - "from-import list not recognised": ("tests.test_g11_census.ToolChecks.test_documented_and_exercised_by_path_import_and_basename", 'self.assertTrue(report[path]["exercised"])'), + "import not recognised": ("tests.test_g11_census.ToolChecks.test_documented_and_exercised_by_path_import_and_basename", 'self.assertTrue(report[path]["referenced"])'), + "from-import list not recognised": ("tests.test_g11_census.ToolChecks.test_documented_and_exercised_by_path_import_and_basename", 'self.assertTrue(report[path]["referenced"])'), "basename not recognised": ("tests.test_g11_census.ToolChecks.test_documented_and_exercised_by_path_import_and_basename", 'self.assertTrue(report[path]["documented"])'), - "transitive exercise off": ("tests.test_g11_census.ToolChecks.test_exercised_through_an_exercised_tool", 'self.assertEqual(report["tools/helper.py"]["exercised_by"],'), + "transitive exercise off": ("tests.test_g11_census.ToolChecks.test_exercised_through_an_exercised_tool", 'self.assertEqual(report["tools/helper.py"]["referenced_by"],'), "site not production": ("tests.test_g11_census.Markers.test_site_marker_fails_and_vendor_marker_does_not", 'self.assertEqual(run({"docs/site/app.mjs":'), "research counted as production": ("tests.test_g11_census.PythonStubs.test_production_stub_fails_but_research_stub_does_not", 'self.assertEqual(research["status"],'), "vendor counted as site": ("tests.test_g11_census.Markers.test_site_marker_fails_and_vendor_marker_does_not", 'self.assertEqual(run({"docs/site/vendor/lib.js":'), "undecodable decoded leniently": ("tests.test_g11_census.FailClosed.test_undecodable_and_unparseable_production_files_fail", 'self.assertEqual(report["status"], "FAIL")'), "unparseable not a failure class": ("tests.test_g11_census.FailClosed.test_undecodable_and_unparseable_production_files_fail", 'self.assertEqual(report["status"], "FAIL")'), - "legitimate context still fails": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", 'self.assertEqual(report["failing"], [])'), + "legitimate context still fails": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", 'self.assertEqual(sorted(f["detail"] for f in report["failing"]), ["__init__: pass", "m: ellipsis"])'), "markers do not fail": ("tests.test_g11_census.Markers.test_production_marker_fails", 'self.assertEqual(report["status"], "FAIL")'), "noop steps do not fail": ("tests.test_g11_census.Workflows.test_production_noop_fails", 'self.assertEqual(run({".github/workflows/w.yml": " - run: true'), "tool findings do not fail": ("tests.test_g11_census.ToolChecks.test_undocumented_and_unexercised_tools_fail", 'self.assertEqual(result["status"], "FAIL")'), @@ -177,25 +179,25 @@ class HarnessError(Exception): NEW_MUTATIONS = [ ("decoy decorator names accepted", "census", " if _trusted_decorator(dec, imports, scope_shadowed):", " if _decorator_name(dec) in LEGIT_DECORATORS:", "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", 'self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]),'), - ("decoy abstract-base names accepted", "census", " if _trusted_abstract_base(base, imports):", " if _base_name(base) in LEGIT_BASES:", + ("decoy abstract-base names accepted", "census", ' if base_kind == "Protocol":', " if _base_name(base) in LEGIT_BASES:", "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", 'self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]),'), - ("decoy exception suffix accepted", "census", " if _builtin_exception_base(base, imports):", + ("decoy exception suffix accepted", "census", " if _builtin_exception_base(base, imports, scope_shadowed):", ' if _base_name(base).endswith(("Error", "Exception", "Warning")):', "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", 'self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]),'), ("decoy request-handler suffix accepted", "census", ' if _http_request_handler_base(base, imports) and func.name == "log_message":', ' if _base_name(base).endswith("RequestHandler") and func.name == "log_message":', "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", 'self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]),'), ("comment and string mentions count as exercise", "census", - "def python_tool_references(text, tools, unique):\n references = set()", - "def python_tool_references(text, tools, unique):\n references = {p for p in tools if p in text}", + 'def python_tool_references(text, tools, unique, mode="script"):\n references = set()', + 'def python_tool_references(text, tools, unique, mode="script"):\n references = {p for p in tools if p in text}', "tests.test_g11_census.AdversarialControls.test_comment_and_string_mentions_do_not_exercise_tools", - 'self.assertFalse(report["tools"]["tools/runner.py"]["exercised"])'), + 'self.assertFalse(report["tools"]["tools/runner.py"]["referenced"])'), ("empty workflow run block not detected", "census", " if not commands:", " if False:", "tests.test_g11_census.AdversarialControls.test_empty_and_comment_only_run_blocks_fail", "self.assertEqual(found,"), ("YAML comments omitted", "census", ' elif path.endswith((".yml", ".yaml")):', " elif False:", "tests.test_g11_census.AdversarialControls.test_yaml_comments_are_scanned_for_markers", "self.assertEqual(classes(report),"), ("shadowed builtin exception name accepted", "census", - 'elif isinstance(node, ast.Name) and node.id in builtins.__dict__ and node.id not in imports["shadowed"]:', + 'elif (isinstance(node, ast.Name) and node.id in builtins.__dict__\n and node.id not in imports["shadowed"] and node.id not in scope_shadowed):', 'elif isinstance(node, ast.Name) and node.id in builtins.__dict__:', "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", 'self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]),'), @@ -203,25 +205,25 @@ class HarnessError(Exception): "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", "self.assertFalse(g11m.qualifies_kill(wrong_line, expected, 17))"), ("function-local decorator shadow accepted", "census", - " if binding and binding in scope_shadowed:\n return False", - " if False:\n return False", + " if binding and binding in scope_shadowed:\n return None", + " if False:\n return None", "tests.test_g11_census.AdversarialControls.test_decoy_names_do_not_exempt_stubs", 'self.assertEqual([(f["class"], f["detail"]) for f in local_report["findings"]'), ("python -c path mention counts as execution", "census", ' if "-c" in tokens[1:]:\n continue', " if False:\n continue", "tests.test_g11_census.AdversarialControls.test_comment_and_string_mentions_do_not_exercise_tools", - 'self.assertFalse(python_cmd["tools"]["tools/runner.py"]["exercised"])'), + 'self.assertFalse(python_cmd["tools"]["tools/runner.py"]["referenced"])'), ("shell -c path mention counts as execution", "census", ' if "-c" in tokens[1:]:\n continue', " if False:\n continue", "tests.test_g11_census.AdversarialControls.test_comment_and_string_mentions_do_not_exercise_tools", - 'self.assertFalse(shell_cmd["tools"]["tools/runner.py"]["exercised"])'), + 'self.assertFalse(shell_cmd["tools"]["tools/runner.py"]["referenced"])'), ("template literal import text counts as static import", "census", ' if quote == "`":\n if ch == "\\\\" and i + 1 < len(text):\n out.extend((" ", " "))\n i += 2\n continue\n if ch == quote:\n quote = None\n out.append("\\n" if ch == "\\n" else " ")\n i += 1\n continue', ' if quote == "`":\n if ch == "\\\\" and i + 1 < len(text):\n out.extend((" ", " "))\n i += 2\n continue\n if ch == quote:\n quote = None\n out.append(ch)\n i += 1\n continue', "tests.test_g11_census.AdversarialControls.test_comment_and_string_mentions_do_not_exercise_tools", - 'self.assertFalse(js_cmd["tools"]["tools/runner.mjs"]["exercised"])'), + 'self.assertFalse(js_cmd["tools"]["tools/runner.mjs"]["referenced"])'), ("parser drops method identity", "harness", ' result["test_id"] = header.group(2) + "." + header.group(1)', ' result["test_id"] = header.group(2)', @@ -257,7 +259,132 @@ class HarnessError(Exception): 'EXCEPTION = re.compile(r"(?m)^([A-Za-z_][A-Za-z0-9_]*(?:Error|Exception|Warning|Interrupt|Exit|StopIteration)):")', "tests.test_g11_census.MutationAttribution.test_mutation_kill_requires_exact_target_assertion", "self.assertTrue(g11m.qualifies_kill(parsed, expected_parser, 123))"), + ("outer class exemption leaks", "census", + " for cls in classes[-1:]:", " for cls in reversed(classes):", + "tests.test_g11_census.ReviewResidualControls.test_stub_exemptions_are_immediate_and_scope_safe", + 'self.assertIn("inner_abc", stub_names)'), + ("local base shadow accepted", "census", + " if binding and binding in scope_shadowed:\n return None", + " if False:\n return None", + "tests.test_g11_census.ReviewResidualControls.test_stub_exemptions_are_immediate_and_scope_safe", + 'self.assertIn("shadowed_abc", stub_names)'), + ("uncalled helper counts as exercise", "census", + " if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):\n # A definition is not a call: only decorators/defaults execute here.", + " if False:\n # A definition is not a call: only decorators/defaults execute here.", + "tests.test_g11_census.ReviewResidualControls.test_tool_reachability_and_heredoc_boundaries", + 'self.assertFalse(uncalled["tools"]["tools/runner.py"]["referenced"])'), + ("heredoc body counted as command", "census", + " if heredoc:", " if False:", + "tests.test_g11_census.ReviewResidualControls.test_tool_reachability_and_heredoc_boundaries", + 'self.assertFalse(heredoc["tools"]["tools/runner.py"]["referenced"])'), + ("mutation inventory shrink accepted", "harness", + " if names != EXPECTED_MUTATION_NAMES:", " if False:", + "tests.test_g11_census.ReviewResidualControls.test_mutation_inventory_is_sealed", + 'self.assertRaisesRegex(g11m.HarnessError, "mutation inventory")'), + ("hostile Python environment survives", "harness", + " for name in PYTHON_ENVIRONMENT_KEYS:", " for name in ():", + "tests.test_g11_census.ReviewResidualControls.test_mutation_child_environment_is_sanitized", + 'self.assertEqual(outcome.get("status"), "OK")'), + ("in-scope tracked symlink silently skipped", "census", + " if path.endswith(SCANNED) and (kind != b\"blob\" or mode == b\"120000\"):", + " if False:", + "tests.test_g11_census.ReviewResidualControls.test_in_scope_tracked_symlink_is_rejected", + 'self.assertRaisesRegex(ValueError, "in-scope non-regular tracked path")'), + ] +EXPECTED_MUTATION_NAMES = ( + "pass not detected", + "ellipsis not detected", + "NotImplementedError not detected", + "raise call not unwrapped", + "docstring-only not reported", + "docstring not stripped", + "decorator context ignored", + "Protocol/ABC context ignored", + "exception context ignored", + "log_message context ignored", + "log_message context too broad", + "module docstring not recorded", + "class bodies not visited", + "nested functions not visited", + "comments read as data", + "strings read as comments", + "marker words case-insensitive", + "marker word boundary dropped", + "quotes not tracked", + "string escapes ignored", + "hash comment needs no whitespace", + "block comments not entered", + "block comment end ignored", + "mentions dropped", + "noop detection off", + "block scalar not read", + "block scalar ends late", + "inline comment kept", + "tests counted as tools", + "__init__ counted as tool", + "basename ambiguity ignored", + "path reference not recognised", + "import not recognised", + "from-import list not recognised", + "basename not recognised", + "transitive exercise off", + "site not production", + "research counted as production", + "vendor counted as site", + "undecodable decoded leniently", + "unparseable not a failure class", + "legitimate context still fails", + "markers do not fail", + "noop steps do not fail", + "tool findings do not fail", + "exit status always zero", + "unreadable revision exits zero", + "decoy decorator names accepted", + "decoy abstract-base names accepted", + "decoy exception suffix accepted", + "decoy request-handler suffix accepted", + "comment and string mentions count as exercise", + "empty workflow run block not detected", + "YAML comments omitted", + "shadowed builtin exception name accepted", + "mutation attribution accepts wrong assertion", + "function-local decorator shadow accepted", + "python -c path mention counts as execution", + "shell -c path mention counts as execution", + "template literal import text counts as static import", + "parser drops method identity", + "wrong test id credited", + "nonfailure status credited", + "error outcome credited", + "skipped outcome credited", + "quoted workflow command boundary ignored", + "FAIL header mistaken for exception", + "outer class exemption leaks", + "local base shadow accepted", + "uncalled helper counts as exercise", + "heredoc body counted as command", + "mutation inventory shrink accepted", + "hostile Python environment survives", + "in-scope tracked symlink silently skipped", +) + + +def _build_cases(): + cases = [] + for name, old, new in MUTATIONS: + target_test, fragment = TARGETS[name] + cases.append((name, "census", old, new, target_test, fragment)) + cases.extend(NEW_MUTATIONS) + return cases + + +def _validate_mutation_inventory(cases): + names = tuple(case[0] for case in cases) + if names != EXPECTED_MUTATION_NAMES: + raise HarnessError("mutation inventory mismatch: expected %d fixed names, received %d" % + (len(EXPECTED_MUTATION_NAMES), len(names))) + def _assertion_line(test_source, test_id, fragment): @@ -328,14 +455,27 @@ def qualifies_kill(outcome, expected_test, expected_line): ) -def _run(workdir, test_id=None): +PYTHON_ENVIRONMENT_KEYS = ("PYTHONPATH", "PYTHONHOME", "PYTHONPYCACHEPREFIX", + "PYTHONSTARTUP", "PYTHONUSERBASE") + + +def _clean_child_env(source=None): + env = dict(os.environ if source is None else source) + for name in PYTHON_ENVIRONMENT_KEYS: + env.pop(name, None) + env["PYTHONNOUSERSITE"] = "1" + return env + + +def _run(workdir, test_id=None, env=None): command = [sys.executable, "-B", "-S"] if sys.flags.optimize: command.append("-O") command.extend(["-m", "unittest"]) command.append(test_id if test_id else "tests.test_g11_census") command.append("-v") - proc = subprocess.run(command, cwd=workdir, capture_output=True, text=True, timeout=300) + proc = subprocess.run(command, cwd=workdir, env=_clean_child_env(env), + capture_output=True, text=True, timeout=300) return _parse_run(proc) @@ -357,14 +497,8 @@ def run_selftest(root): test_source = fh.read() with open(harness_path, encoding="utf-8") as fh: original_harness = fh.read() - cases = [] - for name, old, new in MUTATIONS: - target_test, fragment = TARGETS[name] - cases.append((name, "census", old, new, target_test, fragment)) - cases.extend(NEW_MUTATIONS) - names = [case[0] for case in cases] - if len(names) != len(set(names)): - raise HarnessError("duplicate mutation names") + cases = _build_cases() + _validate_mutation_inventory(cases) for name, target, old, new, test_id, fragment in cases: source = original_tool if target == "census" else original_harness if source.count(old) != 1 or old == new: @@ -377,26 +511,28 @@ def run_selftest(root): base = os.path.join(tmp, "baseline") _sandbox(original_tool, test_source, original_harness, base) baseline = _run(base) - for index, (name, target, old, new, test_id, fragment) in enumerate(cases): - work = os.path.join(tmp, "m%02d" % index) - tool_source = original_tool - harness_source = original_harness - if target == "census": - tool_source = original_tool.replace(old, new, 1) - else: - harness_source = original_harness.replace(old, new, 1) - _sandbox(tool_source, test_source, harness_source, work) - outcome = _run(work, test_id) - expected_line = _assertion_line(test_source, test_id, fragment) - results.append(dict(outcome, mutation=name, target=target, expected_test=test_id, - expected_line=expected_line, - killed=qualifies_kill(outcome, test_id, expected_line))) - baseline_ok = (baseline.get("parsed") and baseline.get("exit") == 0 and baseline.get("ran") == 38 - and baseline.get("failures") == 0 and baseline.get("errors") == 0 - and baseline.get("skipped") == 0 and baseline.get("status") == "OK") + baseline_ok = (baseline.get("parsed") and baseline.get("exit") == 0 and baseline.get("ran") == 38 + and baseline.get("failures") == 0 and baseline.get("errors") == 0 + and baseline.get("skipped") == 0 and baseline.get("status") == "OK") + if baseline_ok: + for index, (name, target, old, new, test_id, fragment) in enumerate(cases): + work = os.path.join(tmp, "m%02d" % index) + tool_source = original_tool + harness_source = original_harness + if target == "census": + tool_source = original_tool.replace(old, new, 1) + else: + harness_source = original_harness.replace(old, new, 1) + _sandbox(tool_source, test_source, original_harness, work) + outcome = _run(work, test_id) + expected_line = _assertion_line(test_source, test_id, fragment) + results.append(dict(outcome, mutation=name, target=target, expected_test=test_id, + expected_line=expected_line, + killed=qualifies_kill(outcome, test_id, expected_line))) killed = sum(r["killed"] for r in results) - return {"baseline": baseline, "mutations": results, "killed": killed, "total": len(results), - "status": "PASS" if baseline_ok and killed == len(results) else "FAIL"} + return {"baseline": baseline, "mutations": results, "killed": killed, "total": len(cases), + "executed": len(results), + "status": "PASS" if baseline_ok and len(results) == len(cases) and killed == len(results) else "FAIL"} def main(argv=None): @@ -416,8 +552,9 @@ def main(argv=None): print("%-8s %s (target %s, exit %d, failures %d, errors %d, skips %d, line %s/%s)" % ("KILLED" if r["killed"] else "SURVIVED", r["mutation"], r["target"], r["exit"], r["failures"], r["errors"], r["skipped"], r["traceback_line"], r["expected_line"])) - print("G11_SELFTEST_%s: baseline %d tests, %d/%d mutations killed" % - (report["status"], report["baseline"].get("ran", -1), report["killed"], report["total"])) + print("G11_SELFTEST_%s: baseline %d tests, %d/%d mutations killed (%d executed)" % + (report["status"], report["baseline"].get("ran", -1), report["killed"], + report["total"], report["executed"])) return 0 if report["status"] == "PASS" else 1 From 2ba97e8c56c847dc238a270797a25ee11dc13565 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:23:30 -0500 Subject: [PATCH 18/27] test(cbz): bind G11 mutations to unique assertions --- tools/cbz/g11_mutation_selftest.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py index 02f5f9e11..0fcc34181 100644 --- a/tools/cbz/g11_mutation_selftest.py +++ b/tools/cbz/g11_mutation_selftest.py @@ -126,11 +126,11 @@ class HarnessError(Exception): TARGETS = { - "pass not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", r'stubs("def a():\n pass\n")'), - "ellipsis not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", r'stubs("def a():\n ...\n")'), - "NotImplementedError not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", r"raise NotImplementedError\n"), - "raise call not unwrapped": ("tests.test_g11_census.PythonStubs.test_each_stub_body", "NotImplementedError('later')"), - "docstring-only not reported": ("tests.test_g11_census.PythonStubs.test_each_stub_body", "Only a docstring."), + "pass not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n pass\\n"), [(1, "a: pass")])'), + "ellipsis not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n ...\\n"), [(1, "a: ellipsis")])'), + "NotImplementedError not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n raise NotImplementedError\\n"), [(1, "a: raise NotImplementedError")])'), + "raise call not unwrapped": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n raise NotImplementedError(\'later\')\\n"), [(1, "a: raise NotImplementedError")])'), + "docstring-only not reported": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs(\'def a():\\n """Only a docstring."""\\n\'), [(1, "a: docstring only")])'), "docstring not stripped": ("tests.test_g11_census.PythonStubs.test_docstring_then_stub_body", "Doc."), "decorator context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), "Protocol/ABC context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), From fec019d18f8b04ba7b8d33a14b1017dd55225478 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:24:28 -0500 Subject: [PATCH 19/27] test(cbz): refine G11 assertion attribution --- tools/cbz/g11_mutation_selftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py index 0fcc34181..d9645a7ce 100644 --- a/tools/cbz/g11_mutation_selftest.py +++ b/tools/cbz/g11_mutation_selftest.py @@ -129,7 +129,7 @@ class HarnessError(Exception): "pass not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n pass\\n"), [(1, "a: pass")])'), "ellipsis not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n ...\\n"), [(1, "a: ellipsis")])'), "NotImplementedError not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n raise NotImplementedError\\n"), [(1, "a: raise NotImplementedError")])'), - "raise call not unwrapped": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n raise NotImplementedError(\'later\')\\n"), [(1, "a: raise NotImplementedError")])'), + "raise call not unwrapped": ("tests.test_g11_census.PythonStubs.test_each_stub_body", "NotImplementedError('later')"), "docstring-only not reported": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs(\'def a():\\n """Only a docstring."""\\n\'), [(1, "a: docstring only")])'), "docstring not stripped": ("tests.test_g11_census.PythonStubs.test_docstring_then_stub_body", "Doc."), "decorator context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), From 9a063fdf3182cb2b53ca8b9d835b763e10adffc3 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:25:28 -0500 Subject: [PATCH 20/27] test(cbz): bind raise-call mutant to full assertion --- tools/cbz/g11_mutation_selftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py index d9645a7ce..5e0f680f2 100644 --- a/tools/cbz/g11_mutation_selftest.py +++ b/tools/cbz/g11_mutation_selftest.py @@ -129,7 +129,7 @@ class HarnessError(Exception): "pass not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n pass\\n"), [(1, "a: pass")])'), "ellipsis not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n ...\\n"), [(1, "a: ellipsis")])'), "NotImplementedError not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n raise NotImplementedError\\n"), [(1, "a: raise NotImplementedError")])'), - "raise call not unwrapped": ("tests.test_g11_census.PythonStubs.test_each_stub_body", "NotImplementedError('later')"), + "raise call not unwrapped": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\n raise NotImplementedError(\'later\')\n"),\n [(1, "a: raise NotImplementedError")])'), "docstring-only not reported": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs(\'def a():\\n """Only a docstring."""\\n\'), [(1, "a: docstring only")])'), "docstring not stripped": ("tests.test_g11_census.PythonStubs.test_docstring_then_stub_body", "Doc."), "decorator context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), From 9dbc9760001c1a142af73ab4ddeffa97fb8730a4 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Fri, 30 Oct 2026 03:01:48 -0500 Subject: [PATCH 21/27] fix(cbz): close G11 auditor soundness and mutation-campaign gaps --- docs/cbz/G11_AUDITOR.md | 6 +-- tests/test_g11_census.py | 67 ++++++++++++++++++++++++--- tools/cbz/g11_census.py | 74 +++++++++++++++++++++++++----- tools/cbz/g11_mutation_selftest.py | 48 ++++++++++++++++--- 4 files changed, 167 insertions(+), 28 deletions(-) diff --git a/docs/cbz/G11_AUDITOR.md b/docs/cbz/G11_AUDITOR.md index 335e6f17f..1b9bf76a1 100644 --- a/docs/cbz/G11_AUDITOR.md +++ b/docs/cbz/G11_AUDITOR.md @@ -20,10 +20,10 @@ The census exits 0 for no detected production finding, 1 for a detected producti ## Soundness boundaries -- Stub exemptions are bound to imported `abc.abstractmethod`, `typing.overload`, `typing_extensions.overload`, immediate `typing.Protocol`/`typing_extensions.Protocol` interface methods, and the named `http.server` request-handler `log_message` override. ABC inheritance alone and exception-class inheritance do not exempt a method. Nested outer-class context, name suffixes, local look-alikes, and lexical bindings shadowing an imported decorator/base do not grant exemptions. -- Python reference evidence is limited to module-level statements and `unittest.TestCase` methods, following only locally reachable helper calls and literal branches; uncalled helpers, `if False`, runtime-dependent branches, and loop bodies are not credited. Workflow references ignore here-document bodies and are read from run commands, not YAML comments. Python/shell `-c` payloads and JavaScript template-literal text are not credited. These are static references only, never proof of runtime execution, and cannot satisfy a runtime-exercise requirement. +- Stub exemptions are bound to source-resolved imports, not decorator or base names. An imported `abc.abstractmethod` exempts only a method in an immediate class with a trusted imported `abc.ABC` base. Imported `typing.overload`/`typing_extensions.overload` exempts only declarations in one contiguous same-name overload group followed by a concrete, non-stub implementation. Immediate imported `typing.Protocol`/`typing_extensions.Protocol` interface methods and the named `http.server` request-handler `log_message` override remain contextual. ABC inheritance alone and exception-class inheritance do not exempt a method. Nested outer-class context, unpaired overloads, name suffixes, local look-alikes, and lexical bindings shadowing an imported decorator/base do not grant exemptions. +- Python reference evidence is limited to module-level statements and classes whose base resolves to the imported `unittest.TestCase` identity (including explicit aliases), following only locally reachable helper calls and literal branches; a local class merely named `TestCase` is not credited. Uncalled helpers, `if False`, runtime-dependent branches, and loop bodies are not credited. Workflow references ignore here-document bodies and are read from run commands, not YAML comments. Python/shell `-c` payloads and JavaScript template-literal text are not credited. These are static references only, never proof of runtime execution, and cannot satisfy a runtime-exercise requirement. - YAML comments are scanned for marker words. Empty and comment-only `run` blocks are reported as no-op steps. -- The mutation self-test preserves the original mutation inventory and admits a kill only when the clean full 38-test baseline succeeds and the exact designated module/class/method reports `FAILED` with one AssertionError at its exact designated assertion line, zero errors, and zero skips. The harness propagates the parent's optimized mode to child tests and mutates its own parser and each attribution guard. +- The mutation self-test preserves all 74 original cases and adds one F4 decoy-`TestCase` control (75 total). It seals the exact source slice containing `MUTATIONS`, `TARGETS`, `NEW_MUTATIONS` and `EXPECTED_MUTATION_NAMES` with a test-side SHA-256, then compares complete literal case tuples—not names alone—against the harness source. A kill is admitted only when the clean full 38-test baseline succeeds and the exact designated module/class/method reports `FAILED` with one AssertionError at its exact designated assertion line, zero errors, and zero skips. The harness propagates the parent's optimized mode to child tests and applies each mutation to its declared source, including all harness-target cases. ## Limits diff --git a/tests/test_g11_census.py b/tests/test_g11_census.py index 23a479702..aea4eada0 100644 --- a/tests/test_g11_census.py +++ b/tests/test_g11_census.py @@ -4,6 +4,7 @@ must not be flagged; the mutation self-test (tools/cbz/g11_mutation_selftest.py) requires these tests to fail when any detector is weakened. """ +import hashlib import os import subprocess import sys @@ -16,6 +17,8 @@ import g11_census as g11 # noqa: E402 import g11_mutation_selftest as g11m # noqa: E402 +EXPECTED_MUTATION_INVENTORY_SHA256 = "30517cb4c2a241620a7f100cd3015f24c157c91ae2eb69dce332db6ddfa2c104" + DOCUMENTED_TESTED = { "tools/README.md": b"Run `tools/good.py` to check things.\n", "tests/test_good.py": b"import good\n", @@ -81,12 +84,15 @@ def test_legitimate_contexts_are_reported_but_do_not_fail(self): "class Base(ABC):\n def m(self):\n ...\n" "class Shape(Protocol):\n def area(self) -> float:\n ...\n" "class Bad(ValueError):\n def __init__(self):\n pass\n" - "class C:\n @abstractmethod\n def a(self):\n pass\n" - " @overload\n def f(self, x: int) -> int:\n ...\n") + "class C(ABC):\n @abstractmethod\n def a(self):\n pass\n" + " @overload\n def f(self, x: int) -> int:\n ...\n" + " @overload\n def f(self, x: str) -> str:\n ...\n" + " def f(self, x):\n return x\n") report = run({"tools/good.py": text, **DOCUMENTED_TESTED}) contexts = sorted(f["detail"] for f in report["findings"] if f["class"] == "stub-context") self.assertEqual(contexts, ["a: pass; @abstractmethod", - "area: ellipsis; method of a Protocol class", "f: ellipsis; @overload"]) + "area: ellipsis; method of a Protocol class", + "f: ellipsis; @overload", "f: ellipsis; @overload"]) self.assertEqual(sorted(f["detail"] for f in report["failing"]), ["__init__: pass", "m: ellipsis"]) self.assertEqual(report["status"], "FAIL") @@ -268,6 +274,13 @@ def test_exit_codes_and_exact_committed_bytes(self): result = self.cli(bad) self.assertEqual(result.returncode, 1) self.assertIn("tools/good.py:1: tooling stub (a: pass)", result.stdout) + unexercised = self.repo({ + "tools/lonely.py": b"x = 1\n", + "tools/README.md": b"tools/lonely.py\n", + }) + unexercised_result = self.cli(unexercised) + self.assertEqual(unexercised_result.returncode, 1) + self.assertIn("tools/lonely.py: tool unexercised", unexercised_result.stdout) self.assertEqual(self.cli(root, "--rev", "no-such-revision").returncode, 2) @@ -294,7 +307,8 @@ def test_decoy_names_do_not_exempt_stubs(self): ) report = run({"tools/good.py": text, **DOCUMENTED_TESTED}) self.assertEqual(sorted(f["detail"].split(":")[0] for f in report["failing"]), - ["abc", "builtin", "decoy_decorator", "decoy_qualified", "fail", "fail", "fail", "log_message", "proto"]) + ["abc", "abstract", "builtin", "decoy_decorator", "decoy_qualified", + "fail", "fail", "fail", "log_message", "proto"]) self.assertEqual(report["status"], "FAIL") local_shadow = ( "from abc import abstractmethod\n" @@ -399,7 +413,7 @@ class ReviewResidualControls(unittest.TestCase): def test_stub_exemptions_are_immediate_and_scope_safe(self): text = ( "from abc import ABC, abstractmethod\n" - "from typing import Protocol\n" + "from typing import Protocol, overload\n" "from http.server import BaseHTTPRequestHandler\n" "class Abstract(ABC):\n" " @abstractmethod\n" @@ -420,6 +434,11 @@ def test_stub_exemptions_are_immediate_and_scope_safe(self): " def shadowed_exception(self): ...\n" " class ShadowedHandler(BaseHTTPRequestHandler):\n" " def shadowed_handler_log_message(self, *args): ...\n" + "class Unpaired:\n" + " @abstractmethod\n" + " def abstract_without_abc(self): ...\n" + " @overload\n" + " def overload_without_implementation(self, value: int): ...\n" ) report = run({"tools/good.py": text, **DOCUMENTED_TESTED}) stub_names = {f["detail"].split(":")[0] for f in report["findings"] if f["class"] == "stub"} @@ -429,7 +448,8 @@ def test_stub_exemptions_are_immediate_and_scope_safe(self): self.assertIn("inner_abc", stub_names) self.assertIn("shadowed_abc", stub_names) for name in ("concrete", "inner_abc", "inner_protocol", "exception_method", - "shadowed_abc", "shadowed_exception", "shadowed_handler_log_message"): + "shadowed_abc", "shadowed_exception", "shadowed_handler_log_message", + "abstract_without_abc", "overload_without_implementation"): self.assertIn(name, stub_names) self.assertNotIn(name, context_names) @@ -458,6 +478,28 @@ def test_tool_reachability_and_heredoc_boundaries(self): ), }) self.assertTrue(reachable["tools"]["tools/runner.py"]["referenced"]) + decoy = run({ + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + "tests/test_runner.py": ( + "class TestCase:\n pass\n" + "class Fake(TestCase):\n" + " def test_decoy(self):\n" + " import runner\n" + ), + }) + self.assertFalse(decoy["tools"]["tools/runner.py"]["referenced"]) + imported_alias = run({ + "tools/runner.py": "def main(): return 0\n", + "tools/README.md": "Use tools/runner.py.\n", + "tests/test_runner.py": ( + "from unittest import TestCase as BaseCase\n" + "class RunnerTests(BaseCase):\n" + " def test_reaches_helper(self):\n" + " import runner\n" + ), + }) + self.assertTrue(imported_alias["tools"]["tools/runner.py"]["referenced"]) workflow = ("jobs:\n test:\n steps:\n - run: |\n" " cat <<'EOF'\n" " python tools/runner.py\n" @@ -476,13 +518,24 @@ def test_tool_reachability_and_heredoc_boundaries(self): self.assertTrue(real_command["tools"]["tools/runner.py"]["referenced"]) def test_mutation_inventory_is_sealed(self): + source = Path(g11m.__file__).read_bytes() + start = source.index(b"MUTATIONS = [") + end = source.index(b"\n\n\ndef _build_cases", start) + self.assertEqual(hashlib.sha256(source[start:end]).hexdigest(), + EXPECTED_MUTATION_INVENTORY_SHA256) cases = g11m._build_cases() - self.assertEqual(len(cases), 74) + self.assertEqual(len(cases), 75) g11m._validate_mutation_inventory(cases) altered_cases = (cases[:-1], cases + [cases[0]], cases[1:] + cases[:1]) for altered in altered_cases: with self.assertRaisesRegex(g11m.HarnessError, "mutation inventory"): g11m._validate_mutation_inventory(altered) + weakened = list(cases) + original = weakened[0] + weakened[0] = (original[0], original[1], original[2], + original[3] + " # same-name weakening", original[4], original[5]) + with self.assertRaisesRegex(g11m.HarnessError, "mutation inventory mismatch"): + g11m._validate_mutation_inventory(tuple(weakened)) def test_mutation_child_environment_is_sanitized(self): with tempfile.TemporaryDirectory() as temporary: diff --git a/tools/cbz/g11_census.py b/tools/cbz/g11_census.py index bb0e62587..d2952d559 100644 --- a/tools/cbz/g11_census.py +++ b/tools/cbz/g11_census.py @@ -5,8 +5,10 @@ stub a Python function whose body, after any docstring, is only `pass`, `...` or `raise NotImplementedError`, or is the docstring alone. - @abstractmethod/@overload and immediate Protocol methods are - contextual; ABC or exception inheritance alone is not an exemption. + @abstractmethod is contextual only in an immediate imported ABC subclass; + @overload requires a complete overload group and a concrete implementation; + immediate Protocol methods are contextual; ABC or exception inheritance alone + is not an exemption. marker TODO, FIXME, XXX, TBD or STUB (upper case) or "not implemented" in a code comment. In Python the comment is found by tokenizing; in .mjs/.js/.cjs/.sh a comment is found by a quote-parity heuristic. @@ -199,6 +201,43 @@ def _trusted_decorator(node, imports, scope_shadowed=()): ("typing_extensions", "overload")} +def _overload_declaration(node, imports, scope_shadowed=()): + return ( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and any(_resolve_scoped_import(dec, imports, scope_shadowed) in { + ("typing", "overload"), ("typing_extensions", "overload") + } for dec in node.decorator_list) + ) + + +def _complete_overload_group(func, siblings, imports, scope_shadowed=()): + if not _overload_declaration(func, imports, scope_shadowed): + return False + try: + index = next(i for i, node in enumerate(siblings) if node is func) + except StopIteration: + return False + start = index + while (start > 0 and isinstance(siblings[start - 1], (ast.FunctionDef, ast.AsyncFunctionDef)) + and siblings[start - 1].name == func.name + and _overload_declaration(siblings[start - 1], imports, scope_shadowed)): + start -= 1 + end = start + while (end < len(siblings) and isinstance(siblings[end], (ast.FunctionDef, ast.AsyncFunctionDef)) + and siblings[end].name == func.name + and _overload_declaration(siblings[end], imports, scope_shadowed)): + end += 1 + if not any(siblings[i] is func for i in range(start, end)) or end >= len(siblings): + return False + implementation = siblings[end] + return ( + isinstance(implementation, (ast.FunctionDef, ast.AsyncFunctionDef)) + and implementation.name == func.name + and not _overload_declaration(implementation, imports, scope_shadowed) + and _stub_kind(implementation.body) is None + ) + + def _trusted_abstract_base(node, imports, scope_shadowed=()): resolved = _resolve_scoped_import(node, imports, scope_shadowed) if resolved == ("abc", "ABC"): @@ -253,10 +292,17 @@ def _stub_kind(body): return None -def _legit_context(func, classes, imports, scope_shadowed=()): +def _legit_context(func, classes, imports, scope_shadowed=(), siblings=()): for dec in func.decorator_list: if _trusted_decorator(dec, imports, scope_shadowed): - return "@" + _decorator_name(dec) + resolved = _resolve_scoped_import(dec, imports, scope_shadowed) + if resolved == ("abc", "abstractmethod"): + if classes and any(_trusted_abstract_base(base, imports, scope_shadowed) == "ABC" + for base in classes[-1].bases): + return "@abstractmethod" + elif (resolved in {("typing", "overload"), ("typing_extensions", "overload")} + and _complete_overload_group(func, siblings, imports, scope_shadowed)): + return "@overload" # Only the immediate class can supply method context. for cls in classes[-1:]: for base in cls.bases: @@ -278,19 +324,21 @@ def python_stubs(tree, imports=None): if imports is None: imports = _import_bindings(tree) - def visit(node, classes, scope_shadowed): + def visit(node, classes, scope_shadowed, siblings): for child in ast.iter_child_nodes(node): if isinstance(child, ast.ClassDef): - visit(child, classes + [child], scope_shadowed | _scope_bindings(child)) + visit(child, classes + [child], scope_shadowed | _scope_bindings(child), child.body) elif isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): kind = _stub_kind(child.body) if kind: found.append({"line": child.lineno, "name": child.name, "kind": kind, - "context": _legit_context(child, classes, imports, scope_shadowed)}) - visit(child, [], scope_shadowed | _scope_bindings(child)) + "context": _legit_context(child, classes, imports, scope_shadowed, siblings)}) + visit(child, [], scope_shadowed | _scope_bindings(child), child.body) else: - visit(child, classes, scope_shadowed) - visit(tree, [], set()) + child_body = getattr(child, "body", None) + visit(child, classes, scope_shadowed, + child_body if isinstance(child_body, list) else siblings) + visit(tree, [], set(), tree.body) return found @@ -529,9 +577,11 @@ def register(body, owner=None): register(tree.body) test_classes = set() if mode == "tests": + imports = _import_bindings(tree) for statement in tree.body: if (isinstance(statement, ast.ClassDef) - and any(_base_name(base) == "TestCase" for base in statement.bases)): + and any(_resolve_import(base, imports) == ("unittest", "TestCase") + for base in statement.bases)): test_classes.add(statement.name) nodes, queue, visited = [], [], set() @@ -947,7 +997,7 @@ def main(argv=None): for path in report["failing_tools"]: t = report["tools"][path] print("%s: tool %s" % (path, ", ".join(w for w, ok in (("undocumented", t["documented"]), - ("unexercised", t["exercised"])) if not ok))) + ("unexercised", t["referenced"])) if not ok))) counts = {} for f in report["findings"]: counts[f["class"]] = counts.get(f["class"], 0) + 1 diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py index 5e0f680f2..0b78bbe6b 100644 --- a/tools/cbz/g11_mutation_selftest.py +++ b/tools/cbz/g11_mutation_selftest.py @@ -278,7 +278,7 @@ class HarnessError(Exception): "tests.test_g11_census.ReviewResidualControls.test_tool_reachability_and_heredoc_boundaries", 'self.assertFalse(heredoc["tools"]["tools/runner.py"]["referenced"])'), ("mutation inventory shrink accepted", "harness", - " if names != EXPECTED_MUTATION_NAMES:", " if False:", + "\n if tuple(cases) != declared:\n raise HarnessError(\"mutation inventory mismatch: complete case tuples differ\")\n", "\n if False:\n raise HarnessError(\"mutation inventory mismatch: complete case tuples differ\")\n", "tests.test_g11_census.ReviewResidualControls.test_mutation_inventory_is_sealed", 'self.assertRaisesRegex(g11m.HarnessError, "mutation inventory")'), ("hostile Python environment survives", "harness", @@ -290,6 +290,11 @@ class HarnessError(Exception): " if False:", "tests.test_g11_census.ReviewResidualControls.test_in_scope_tracked_symlink_is_rejected", 'self.assertRaisesRegex(ValueError, "in-scope non-regular tracked path")'), + ("decoy TestCase base credited", "census", + 'and any(_resolve_import(base, imports) == ("unittest", "TestCase")', + 'and any(_base_name(base) == "TestCase"', + "tests.test_g11_census.ReviewResidualControls.test_tool_reachability_and_heredoc_boundaries", + 'self.assertFalse(decoy["tools"]["tools/runner.py"]["referenced"])'), ] EXPECTED_MUTATION_NAMES = ( @@ -367,6 +372,7 @@ class HarnessError(Exception): "mutation inventory shrink accepted", "hostile Python environment survives", "in-scope tracked symlink silently skipped", + "decoy TestCase base credited", ) @@ -379,11 +385,41 @@ def _build_cases(): return cases +def _declared_mutation_cases(source): + tree = ast.parse(source) + required = {"MUTATIONS", "TARGETS", "NEW_MUTATIONS", "EXPECTED_MUTATION_NAMES"} + values = {} + for node in tree.body: + if (isinstance(node, ast.Assign) and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id in required): + values[node.targets[0].id] = ast.literal_eval(node.value) + if set(values) != required: + raise HarnessError("mutation inventory declarations are incomplete") + cases = [] + for item in values["MUTATIONS"]: + if not isinstance(item, (tuple, list)) or len(item) != 3: + raise HarnessError("mutation inventory declaration has an invalid census tuple") + name, old, new = item + try: + target_test, fragment = values["TARGETS"][name] + except (KeyError, TypeError, ValueError) as exc: + raise HarnessError("mutation inventory target missing for %r: %s" % (name, exc)) + cases.append((name, "census", old, new, target_test, fragment)) + cases.extend(values["NEW_MUTATIONS"]) + expected_names = tuple(values["EXPECTED_MUTATION_NAMES"]) + if tuple(case[0] for case in cases) != expected_names: + raise HarnessError("mutation inventory names disagree with literal declarations") + return tuple(cases) + + def _validate_mutation_inventory(cases): - names = tuple(case[0] for case in cases) - if names != EXPECTED_MUTATION_NAMES: - raise HarnessError("mutation inventory mismatch: expected %d fixed names, received %d" % - (len(EXPECTED_MUTATION_NAMES), len(names))) + try: + declared = _declared_mutation_cases(Path(__file__).read_text(encoding="utf-8")) + except (OSError, SyntaxError, ValueError, TypeError, KeyError) as exc: + raise HarnessError("mutation inventory cannot be parsed: %s" % exc) + if tuple(cases) != declared: + raise HarnessError("mutation inventory mismatch: complete case tuples differ") @@ -523,7 +559,7 @@ def run_selftest(root): tool_source = original_tool.replace(old, new, 1) else: harness_source = original_harness.replace(old, new, 1) - _sandbox(tool_source, test_source, original_harness, work) + _sandbox(tool_source, test_source, harness_source, work) outcome = _run(work, test_id) expected_line = _assertion_line(test_source, test_id, fragment) results.append(dict(outcome, mutation=name, target=target, expected_test=test_id, From 51293d15dd5cc2cf76ad84548c686c6baed26498 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Fri, 30 Oct 2026 03:03:53 -0500 Subject: [PATCH 22/27] test(cbz): repair exact mutation assertion binding --- tests/test_g11_census.py | 2 +- tools/cbz/g11_mutation_selftest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_g11_census.py b/tests/test_g11_census.py index aea4eada0..962f67ac2 100644 --- a/tests/test_g11_census.py +++ b/tests/test_g11_census.py @@ -17,7 +17,7 @@ import g11_census as g11 # noqa: E402 import g11_mutation_selftest as g11m # noqa: E402 -EXPECTED_MUTATION_INVENTORY_SHA256 = "30517cb4c2a241620a7f100cd3015f24c157c91ae2eb69dce332db6ddfa2c104" +EXPECTED_MUTATION_INVENTORY_SHA256 = "60ee7002c7a5633e4c658113c82b40a81ff76a5a0707aff79da9b77eccc35b42" DOCUMENTED_TESTED = { "tools/README.md": b"Run `tools/good.py` to check things.\n", diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py index 0b78bbe6b..00e4bda42 100644 --- a/tools/cbz/g11_mutation_selftest.py +++ b/tools/cbz/g11_mutation_selftest.py @@ -129,7 +129,7 @@ class HarnessError(Exception): "pass not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n pass\\n"), [(1, "a: pass")])'), "ellipsis not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n ...\\n"), [(1, "a: ellipsis")])'), "NotImplementedError not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n raise NotImplementedError\\n"), [(1, "a: raise NotImplementedError")])'), - "raise call not unwrapped": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\n raise NotImplementedError(\'later\')\n"),\n [(1, "a: raise NotImplementedError")])'), + "raise call not unwrapped": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n raise NotImplementedError(\'later\')\\n"),\\n [(1, "a: raise NotImplementedError")])'), "docstring-only not reported": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs(\'def a():\\n """Only a docstring."""\\n\'), [(1, "a: docstring only")])'), "docstring not stripped": ("tests.test_g11_census.PythonStubs.test_docstring_then_stub_body", "Doc."), "decorator context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), From 3433a6701aea670ce54528dc43d63d9638732787 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Fri, 30 Oct 2026 03:05:06 -0500 Subject: [PATCH 23/27] test(cbz): align multiline mutation assertion source binding --- tests/test_g11_census.py | 2 +- tools/cbz/g11_mutation_selftest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_g11_census.py b/tests/test_g11_census.py index 962f67ac2..be56ab893 100644 --- a/tests/test_g11_census.py +++ b/tests/test_g11_census.py @@ -17,7 +17,7 @@ import g11_census as g11 # noqa: E402 import g11_mutation_selftest as g11m # noqa: E402 -EXPECTED_MUTATION_INVENTORY_SHA256 = "60ee7002c7a5633e4c658113c82b40a81ff76a5a0707aff79da9b77eccc35b42" +EXPECTED_MUTATION_INVENTORY_SHA256 = "1034b0b85454441e69dc8c8d14b32f1fabbf72f998812749b563b9f4c5d6ec65" DOCUMENTED_TESTED = { "tools/README.md": b"Run `tools/good.py` to check things.\n", diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py index 00e4bda42..bafbc46f9 100644 --- a/tools/cbz/g11_mutation_selftest.py +++ b/tools/cbz/g11_mutation_selftest.py @@ -129,7 +129,7 @@ class HarnessError(Exception): "pass not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n pass\\n"), [(1, "a: pass")])'), "ellipsis not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n ...\\n"), [(1, "a: ellipsis")])'), "NotImplementedError not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n raise NotImplementedError\\n"), [(1, "a: raise NotImplementedError")])'), - "raise call not unwrapped": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n raise NotImplementedError(\'later\')\\n"),\\n [(1, "a: raise NotImplementedError")])'), + "raise call not unwrapped": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n raise NotImplementedError(\'later\')\\n"),\n [(1, "a: raise NotImplementedError")])'), "docstring-only not reported": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs(\'def a():\\n """Only a docstring."""\\n\'), [(1, "a: docstring only")])'), "docstring not stripped": ("tests.test_g11_census.PythonStubs.test_docstring_then_stub_body", "Doc."), "decorator context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), From 3901886fed22cf802c54435794c538af3b05c5cb Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Fri, 30 Oct 2026 03:06:04 -0500 Subject: [PATCH 24/27] test(cbz): bind docstring mutant to unique assertion --- tests/test_g11_census.py | 2 +- tools/cbz/g11_mutation_selftest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_g11_census.py b/tests/test_g11_census.py index be56ab893..c7e851728 100644 --- a/tests/test_g11_census.py +++ b/tests/test_g11_census.py @@ -17,7 +17,7 @@ import g11_census as g11 # noqa: E402 import g11_mutation_selftest as g11m # noqa: E402 -EXPECTED_MUTATION_INVENTORY_SHA256 = "1034b0b85454441e69dc8c8d14b32f1fabbf72f998812749b563b9f4c5d6ec65" +EXPECTED_MUTATION_INVENTORY_SHA256 = "b5adde1f0b31464b1b27cd5865976ada8cc77ab5ccb593fde94668bb1358a632" DOCUMENTED_TESTED = { "tools/README.md": b"Run `tools/good.py` to check things.\n", diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py index bafbc46f9..dc224aeff 100644 --- a/tools/cbz/g11_mutation_selftest.py +++ b/tools/cbz/g11_mutation_selftest.py @@ -131,7 +131,7 @@ class HarnessError(Exception): "NotImplementedError not detected": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n raise NotImplementedError\\n"), [(1, "a: raise NotImplementedError")])'), "raise call not unwrapped": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs("def a():\\n raise NotImplementedError(\'later\')\\n"),\n [(1, "a: raise NotImplementedError")])'), "docstring-only not reported": ("tests.test_g11_census.PythonStubs.test_each_stub_body", 'self.assertEqual(stubs(\'def a():\\n """Only a docstring."""\\n\'), [(1, "a: docstring only")])'), - "docstring not stripped": ("tests.test_g11_census.PythonStubs.test_docstring_then_stub_body", "Doc."), + "docstring not stripped": ("tests.test_g11_census.PythonStubs.test_docstring_then_stub_body", '[(1, "a: pass")])'), "decorator context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), "Protocol/ABC context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), "exception context ignored": ("tests.test_g11_census.ReviewResidualControls.test_stub_exemptions_are_immediate_and_scope_safe", 'self.assertIn("exception_method", stub_names)'), From 6a91249d77b4e7e3470cfa38566576f9ba2efb8c Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Fri, 30 Oct 2026 03:07:14 -0500 Subject: [PATCH 25/27] test(cbz): bind exception-context mutant to loop assertion --- tests/test_g11_census.py | 2 +- tools/cbz/g11_mutation_selftest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_g11_census.py b/tests/test_g11_census.py index c7e851728..da504ed6e 100644 --- a/tests/test_g11_census.py +++ b/tests/test_g11_census.py @@ -17,7 +17,7 @@ import g11_census as g11 # noqa: E402 import g11_mutation_selftest as g11m # noqa: E402 -EXPECTED_MUTATION_INVENTORY_SHA256 = "b5adde1f0b31464b1b27cd5865976ada8cc77ab5ccb593fde94668bb1358a632" +EXPECTED_MUTATION_INVENTORY_SHA256 = "d1f564e77ab8446b9410e84f77b4d966a932a647335bcdb63ff4101b000d40a6" DOCUMENTED_TESTED = { "tools/README.md": b"Run `tools/good.py` to check things.\n", diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py index dc224aeff..e53589a88 100644 --- a/tools/cbz/g11_mutation_selftest.py +++ b/tools/cbz/g11_mutation_selftest.py @@ -134,7 +134,7 @@ class HarnessError(Exception): "docstring not stripped": ("tests.test_g11_census.PythonStubs.test_docstring_then_stub_body", '[(1, "a: pass")])'), "decorator context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), "Protocol/ABC context ignored": ("tests.test_g11_census.PythonStubs.test_legitimate_contexts_are_reported_but_do_not_fail", "self.assertEqual(contexts,"), - "exception context ignored": ("tests.test_g11_census.ReviewResidualControls.test_stub_exemptions_are_immediate_and_scope_safe", 'self.assertIn("exception_method", stub_names)'), + "exception context ignored": ("tests.test_g11_census.ReviewResidualControls.test_stub_exemptions_are_immediate_and_scope_safe", 'self.assertIn(name, stub_names)'), "log_message context ignored": ("tests.test_g11_census.PythonStubs.test_request_handler_log_message_override_is_legitimate", "self.assertEqual(found,"), "log_message context too broad": ("tests.test_g11_census.PythonStubs.test_request_handler_log_message_override_is_legitimate", "self.assertEqual(found,"), "module docstring not recorded": ("tests.test_g11_census.ToolChecks.test_module_docstring_is_recorded", "self.assertEqual((report["), From 1b96460453304b8db631a3dcee0a0c2a5bd4b60a Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Fri, 30 Oct 2026 05:18:01 -0500 Subject: [PATCH 26/27] fix(cbz): align G11 mutation targets with current source --- tests/test_g11_census.py | 2 +- tools/cbz/g11_mutation_selftest.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_g11_census.py b/tests/test_g11_census.py index da504ed6e..e4fb44b17 100644 --- a/tests/test_g11_census.py +++ b/tests/test_g11_census.py @@ -17,7 +17,7 @@ import g11_census as g11 # noqa: E402 import g11_mutation_selftest as g11m # noqa: E402 -EXPECTED_MUTATION_INVENTORY_SHA256 = "d1f564e77ab8446b9410e84f77b4d966a932a647335bcdb63ff4101b000d40a6" +EXPECTED_MUTATION_INVENTORY_SHA256 = "10e62de452e4966039e6fee0232989828d40e8855c45e13d4eee06208f034982" DOCUMENTED_TESTED = { "tools/README.md": b"Run `tools/good.py` to check things.\n", diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py index e53589a88..ea7b33dc6 100644 --- a/tools/cbz/g11_mutation_selftest.py +++ b/tools/cbz/g11_mutation_selftest.py @@ -49,8 +49,8 @@ ("exception context ignored", " if _builtin_exception_base(base, imports, scope_shadowed):\n continue\n", " if _builtin_exception_base(base, imports, scope_shadowed):\n return \"method of an exception class\"\n"), - ("log_message context ignored", ' if _http_request_handler_base(base, imports, scope_shadowed) and func.name == "log_message":', " if False:"), - ("log_message context too broad", ' if _http_request_handler_base(base, imports, scope_shadowed) and func.name == "log_message":', + ("log_message context ignored", '" if (_http_request_handler_base(base, imports, scope_shadowed)\n and func.name == \"log_message\"):"', " if False:"), + ("log_message context too broad", '" if (_http_request_handler_base(base, imports, scope_shadowed)\n and func.name == \"log_message\"):"', ' if _http_request_handler_base(base, imports, scope_shadowed):'), ("module docstring not recorded", " docstring = bool(ast.get_docstring(ast.parse(files[path])))", " docstring = True"), From 2aa6b26e1bb0964f4bd9b1a4b640ecd617ef9c95 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Fri, 30 Oct 2026 05:21:36 -0500 Subject: [PATCH 27/27] fix(cbz): preserve log-message mutation target literal --- tests/test_g11_census.py | 2 +- tools/cbz/g11_mutation_selftest.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_g11_census.py b/tests/test_g11_census.py index e4fb44b17..fb400aca9 100644 --- a/tests/test_g11_census.py +++ b/tests/test_g11_census.py @@ -17,7 +17,7 @@ import g11_census as g11 # noqa: E402 import g11_mutation_selftest as g11m # noqa: E402 -EXPECTED_MUTATION_INVENTORY_SHA256 = "10e62de452e4966039e6fee0232989828d40e8855c45e13d4eee06208f034982" +EXPECTED_MUTATION_INVENTORY_SHA256 = "f4e4e5226705746d6ae7625c59efab54410d9fabf7622e11bd99889b2d7aee7a" DOCUMENTED_TESTED = { "tools/README.md": b"Run `tools/good.py` to check things.\n", diff --git a/tools/cbz/g11_mutation_selftest.py b/tools/cbz/g11_mutation_selftest.py index ea7b33dc6..c7f74bb27 100644 --- a/tools/cbz/g11_mutation_selftest.py +++ b/tools/cbz/g11_mutation_selftest.py @@ -49,8 +49,8 @@ ("exception context ignored", " if _builtin_exception_base(base, imports, scope_shadowed):\n continue\n", " if _builtin_exception_base(base, imports, scope_shadowed):\n return \"method of an exception class\"\n"), - ("log_message context ignored", '" if (_http_request_handler_base(base, imports, scope_shadowed)\n and func.name == \"log_message\"):"', " if False:"), - ("log_message context too broad", '" if (_http_request_handler_base(base, imports, scope_shadowed)\n and func.name == \"log_message\"):"', + ("log_message context ignored", ' if (_http_request_handler_base(base, imports, scope_shadowed)\n and func.name == "log_message"):', " if False:"), + ("log_message context too broad", ' if (_http_request_handler_base(base, imports, scope_shadowed)\n and func.name == "log_message"):', ' if _http_request_handler_base(base, imports, scope_shadowed):'), ("module docstring not recorded", " docstring = bool(ast.get_docstring(ast.parse(files[path])))", " docstring = True"),