diff --git a/README.md b/README.md index e28196b..43adf2f 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ Run the full Tier 2 oracle gates and Family `A` to `D` graders. On Linux, `lake ## What is not proved - Passing Tier 2 oracles proves only the machine-checked properties enforced by each checker. Whether a theorem statement matches human intent remains a Tier 1 human specification judgment. -- `datasets/scripts/grade_item.py` grades Family C items by deterministic structural claim-tier and `meta["bar_status"]` replay rather than re-running Lean compilation on prose-only reports. +- `datasets/scripts/grade_item.py` grades Family C items by deterministic structural claim-tier and `meta["bar_status"]` replay. The `bar_check` replay pins `spec_meta.statements` and does not emit an identity scaffold. It does not re-run Lean compilation. `verdict_audit` items, including `PUB-C-V02`, check a verdict JSON block and do not use the Family C Lean repair path. - Live `8` GiB RTX 3080 `T-C5` telemetry (`stack/results/final/ux582hs_rtx3080_tc5_live_receipt.json`) measured Windows 11 NVIDIA Optimus (`driver 566.07`, `0 MiB` idle WDDM VRAM), while bare-metal Linux kernel execution on `UX582HS` remains tracked separately. ## What you must trust diff --git a/bin/bootstrap-deps b/bin/bootstrap-deps index 4a7d967..9f7f8e5 100755 --- a/bin/bootstrap-deps +++ b/bin/bootstrap-deps @@ -30,7 +30,7 @@ from mcproof.lineage import ( # noqa: E402 def _clone_at_exact_sha(url: str, commit: str, dest: Path) -> None: - if not COMMIT_RE.match(commit): + if not COMMIT_RE.fullmatch(commit): raise ValueError(f"refusing non-40-hex commit pin {commit!r} for {url}") if (dest / ".git").exists(): head = subprocess.run( diff --git a/datasets/README.md b/datasets/README.md index a4871bc..18d3e57 100644 --- a/datasets/README.md +++ b/datasets/README.md @@ -26,7 +26,7 @@ All 144 `PUB-` prompts stay strictly inside four reserved public subtopics so th - **Family A (`formal_proof`, 24 compile / 12 validation)**: stock-`Init` Lean `v4.35.0` proofs and `WeakStatement` vacuous-hypothesis traps checked across all six kernels of `lake comparator --paranoid`, `leanchecker`, `collectAxioms`, and `soko` `lean4_auditor.py`. - **Family B (`code`, 24 compile / 12 validation)**: exact `int` and `Fraction` Python certificate verifiers (`def verify(inst, cert) -> bool`) audited for distinct AST alpha-skeletons, zero `float` or `/` coercion, zero `assert` under `mde guard --code-audit`, clean `python3 -O -B -W error` execution, and 100% planted-bug mutation kills (`>= 80%` floor). -- **Family C (`formal_proof`, 24 compile / 12 validation)**: epistemic claim-tier hygiene and refusal traps checked by `mcproof`, `mission/bar/check.py`, `formal-claims-audit/scripts/check_verdicts.py`, and `f0/scripts/gate.py`. +- **Family C (`formal_proof`, 24 compile / 12 validation)**: epistemic claim-tier hygiene and refusal traps checked by `mcproof`, `mission/bar/check.py`, `formal-claims-audit/scripts/check_verdicts.py`, and `f0/scripts/gate.py`. The `bar_check` oracle is the one that builds a Lean project from pinned `spec_meta.statements`. `PUB-C-V02` is a `verdict_audit` item and does not use that path. - **Family D (`formal_proof`, 24 compile / 12 validation)**: mixed-register technical notes combining plain prose (`check-prose.py` and `--dupes`), `
` protection, full 40-character verdict SHAs, kernel-checked Lean 4 blocks, and mandatory boundary sections. ## Re-check Commands diff --git a/datasets/SHA256SUMS b/datasets/SHA256SUMS index d7f80d6..8baa97f 100644 --- a/datasets/SHA256SUMS +++ b/datasets/SHA256SUMS @@ -1,4 +1,4 @@ -a584a5b6254aae9dbf800643dcea3c9786a93198ee6975adea00fcc0eaa56007 README.md +42b9a50198da3641a98deefcc754bca2ce474b453918ea7a274c4ffd73f1f2c9 README.md 067059259c5d2238a8ab3f6a6b01ebcedeed2dc0d19d2f7a02a3e170f52f7f31 compile.jsonl 6d3918122d61d92e74752b4a26f70dbc3e5786657df596b54e54a384cb1c9140 harnesses/family_a/PUB-A-C01.json 3e5993ed8ed763e630129fe2426eadea2e0c03dc48d705de8dcbe75a4faedcec harnesses/family_a/PUB-A-C02.json @@ -146,7 +146,7 @@ dfc1d7b0f63c89861c06453993d6aec79249231272f3200a0c3e378a7131658c harnesses/fami 011e0e35a21fe2cd7e54bb9c4efed1e8208923d35a1110e4ff18b7ce1f51631a harnesses/family_d/PUB-D-V12.json ba2e9b277eeb8d9ea5965b27d73ca8efc6c55c732ad800ee2bdfea41266eeeae hybrid_k0.jsonl 93a27e6c37e501153ff5837abaebec10ff80a85b84dc9c15897809687d822142 manifest.json -1c774b9040622baf59c4d0ca2ab3e49807131d83b66cdb6aa16387ddaf8113e7 scripts/grade_item.py +cd97d19a0c6b773636fb2fab15bd3dff10fcfe242d32aa77b69e8f0d20d05556 scripts/grade_item.py f5cb4f7d69da62e4be17a4947049b89928d1ea82c1a2a93dd7d7851ee623611d targets/family_a/PUB-A-C01.lean 7e688d533d3e502b272bfe1b166d6eceaa7f635ac4c511ca7a1449b591257f69 targets/family_a/PUB-A-C02.lean f1f8e95c36306ca5a9abc45cee000ffe7d783d8f012eb3e9a47e583e6fbee82b targets/family_a/PUB-A-C03.lean diff --git a/datasets/scripts/grade_item.py b/datasets/scripts/grade_item.py index 20b73b1..6548a85 100755 --- a/datasets/scripts/grade_item.py +++ b/datasets/scripts/grade_item.py @@ -4,9 +4,11 @@ Delegates primary Family A/B/C/D checks to `bin/mcproof` and runs structural claim/metadata replay (`mission/bar/check.py` README/bar.toml consistency, verdict JSON field checks, and `f0/scripts/gate.py` feed schema checks) for -Family C items when `--id` is supplied. Note: Family C audits prose/metadata -alignment and claim-tier honesty, whereas Lean 4 theorem proving is tested in -Family A and Family D. +Family C items when `--id` is supplied. A `bar_check` replay pins +`spec_meta.statements` and does not emit an identity scaffold. It does not +compile Lean. A `verdict_audit` item, including `PUB-C-V02`, checks a verdict +JSON block and does not build a Lean project. Lean 4 theorem proving is tested +in Family A and Family D. """ from __future__ import annotations @@ -51,6 +53,17 @@ def _resolve_deps_root() -> Path: MCPROOF_BIN = _resolve_mcproof_bin() DEPS_ROOT = _resolve_deps_root() +if str(REPO_ROOT / "src") not in sys.path: + sys.path.insert(0, str(REPO_ROOT / "src")) + +from mcproof.repair_loop import ( # noqa: E402 + _family_c_lakefile, + _family_c_root_source, + _family_c_spec_source, + _family_c_statement_headers, + _family_c_code_under_proof, +) + def _load_item(item_id: str) -> dict[str, Any]: for name in ("compile.jsonl", "validation.jsonl"): @@ -83,41 +96,35 @@ def _run_family_c_structural_replay(harness: dict[str, Any], completion_text: st errs: list[str] = [] if otype == "bar_check": + pkg = str(meta["pkg"]) + thms = list(meta["theorem_names"]) + headers = _family_c_statement_headers(meta, thms) + if headers is None: + return False, [ + "target_statements_absent: spec_meta.statements must pin the header of every theorem_names entry" + ] bar_tpl = DEPS_ROOT / "mission" / "bar" / "template" bar_chk = DEPS_ROOT / "mission" / "bar" / "check.py" if not bar_tpl.is_dir() or not bar_chk.is_file(): return False, [f"missing_mission_dep_under_{DEPS_ROOT.name}__run_bin_bootstrap_deps"] - pkg = str(meta["pkg"]) - thms = list(meta["theorem_names"]) readme_body = re.sub(r"(?im)^Claim-Tier:[^\n]*\n+", "", completion_text, count=1) # Take bar_status strictly from harness metadata; candidate prose cannot self-promote status. bar_status = str(meta["bar_status"]) + code_toml = ", ".join(json.dumps(name) for name in _family_c_code_under_proof(pkg, meta)) with tempfile.TemporaryDirectory(prefix="mcproof_ds_bar_") as td: root = Path(td) shutil.copytree(bar_tpl, root, dirs_exist_ok=True) (root / "Clamp.lean").unlink(missing_ok=True) shutil.rmtree(root / "Clamp", ignore_errors=True) - (root / f"{pkg}.lean").write_text(f"namespace {pkg}\ndef op (x : Nat) : Nat := x\nend {pkg}\n", encoding="utf-8") + (root / f"{pkg}.lean").write_text(_family_c_root_source(pkg, meta), encoding="utf-8") (root / pkg).mkdir(exist_ok=True) - spec_lines = [f"import {pkg}", f"namespace {pkg}.Spec"] - proof_lines = [f"import {pkg}", f"namespace {pkg}.Spec"] - for tfull in thms: - tshort = tfull.split(".")[-1] - spec_lines.append(f"theorem {tshort} (x : Nat) : {pkg}.op x = x := by sorry") - proof_lines.append(f"theorem {tshort} (x : Nat) : {pkg}.op x = x := rfl") - spec_lines.append(f"end {pkg}.Spec\n") - proof_lines.append(f"end {pkg}.Spec\n") - (root / pkg / "Spec.lean").write_text("\n".join(spec_lines), encoding="utf-8") - (root / pkg / "Proofs.lean").write_text("\n".join(proof_lines), encoding="utf-8") - (root / "Main.lean").write_text(f'import {pkg}\ndef main : IO Unit := IO.println s!"{{{pkg}.op 1}}"\n', encoding="utf-8") - (root / "lakefile.toml").write_text( - f'name = "{pkg.lower()}"\nversion = "0.1.0"\ndefaultTargets = ["{pkg}"]\n\n' - f'[[lean_lib]]\nname = "{pkg}"\n\n' - f'[[lean_lib]]\nname = "{pkg}.Spec"\n\n' - f'[[lean_lib]]\nname = "{pkg}.Proofs"\n\n' - f'[[lean_exe]]\nname = "{pkg.lower()}"\nroot = "Main"\n', + (root / pkg / "Spec.lean").write_text(_family_c_spec_source(pkg, headers), encoding="utf-8") + (root / pkg / "Proofs.lean").write_text(f"import {pkg}\n", encoding="utf-8") + (root / "Main.lean").write_text( + f'import {pkg}\ndef main : IO Unit := IO.println "ok"\n', encoding="utf-8", ) + (root / "lakefile.toml").write_text(_family_c_lakefile(pkg), encoding="utf-8") (root / "comparator.json").write_text( json.dumps( { @@ -134,7 +141,7 @@ def _run_family_c_structural_replay(harness: dict[str, Any], completion_text: st (root / "bar.toml").write_text( f'[project]\nname = "{pkg.lower()}"\nstatus = "{bar_status}"\npage = "README.md"\n' f'spec_authors = ["example-author"]\nentry_points = ["Main.lean"]\n\n' - f'[spec]\nfiles = ["SPEC.md", "{pkg}/Spec.lean"]\ncode = ["{pkg}.op"]\n\n' + f'[spec]\nfiles = ["SPEC.md", "{pkg}/Spec.lean"]\ncode = [{code_toml}]\n\n' f'[tests]\nresults = "tests/TEST-RESULTS.md"\ncommand = "{meta["test_cmd"]}"\nfiles = ["tests/run_tests.py"]\n', encoding="utf-8", ) diff --git a/docs/CONTINUITY.md b/docs/CONTINUITY.md index 5fda783..e386f55 100644 --- a/docs/CONTINUITY.md +++ b/docs/CONTINUITY.md @@ -91,3 +91,15 @@ python3 verification/build_final_receipt.py 1. **Multi-layer `blk.28..31` `TangentBetaEdit` live serving promotion:** Wire 4-layer `TangentBetaEdit` (`weft` `c33bbf4f`) into live `warp-modeld` Stage 1 to 4 admission and measure multi-turn `k = 1..4` pass curves with `blk.28..31` non-uniform depth allocation (`alpha_l*` proportional to `R_l^2 / (gamma_l^2 * rho_l)`). 2. **Concrete graph-hypothesis coupling for `Mcproof.TransitionBounds`:** Mechanize finite-table checkers discharging projection and cut-capacity hypotheses directly inside Lean `v4.35.0` alongside Python `verify_dual_product_parity`. + +## D11 Family C real-Lean repair turn (2026-11-20) +- **Defect closed in `src/mcproof/repair_loop.py`:** the public Family C scorer parsed `says_verified` from candidate prose and rebuilt an identity `rfl` scaffold while running the comparator with `--no-lean`. Candidate Lean never compiled, and prose could self-promote `bar_status`. +- **What uses the Lean path:** `_run_family_c_real_lean_turn` runs only when `oracle_type` is `bar_check`. Public `bar_check` items include `PUB-C-V01` (`CrtPair`) and `PUB-C-C01` (`ModInv`). `bar_status` comes from harness `spec_meta`. A spoofed verified claim yields `FAMILY_C_STATUS_SELF_ATTESTATION`. Missing lean blocks and `sorry` or `admit` fail closed before `lake`. +- **What does not use it:** `PUB-C-V02` is `oracle_type: verdict_audit` in `datasets/harnesses/family_c/PUB-C-V02.json`. That path checks a fenced verdict JSON block. It does not build a Lean project, does not hash a theorem statement, and does not call `lake`. A solve of `PUB-C-V02` is a verdict-audit result. It is not evidence that this Lean path accepted a proof. +- **Statement pin:** `spec_meta.statements` supplies the target header for each fully-qualified theorem name. The guard hashes that header. It does not hash `op x = x`. A fully qualified `rfl` of the identity fails with `FAMILY_C_STATEMENT_MISMATCH`. A missing header fails with `FAMILY_C_TARGET_STATEMENT_ABSENT` and is not replaced by an identity scaffold. +- **`defaultTargets`:** the generated project targets `["{pkg}", "{pkg}.Spec", "{pkg}.Proofs"]`, so `lake build` compiles the candidate proofs. The old `["{pkg}"]` list left broken proofs uncompiled. +- **Lean evidence in this lane:** `test_family_c_known_bad_proof_reaches_lake_build` states the pinned headers and closes them with `exact Nat.noSuchLemma`. The turn records `LAKE_BUILD_FAILED` and keeps the compiler `file:line:column` tail. +- **Skip rule:** that test skips only when `lean` and `lake` are absent, and the skip reason says the lake path was not exercised. `test_family_c_lean_error_text_reaches_next_turn_prompt` only checks `format_diagnostics` on a hand-written finding. It does not run `lake`. +- **`datasets/scripts/grade_item.py`:** the structural bar replay writes those same pinned headers into `Spec.lean`. It does not write an identity `op` or an identity `rfl` proof. `defaultTargets` includes the spec and proofs libraries. Harnesses that omit `statements`, which is every frozen `bar_check` file today, fail closed with `target_statements_absent`. This replay still does not compile Lean. +- **Known wart:** `verify_lean4_project_comparator` still keeps only the last 6 `lake build` lines. The repair turn records its own `lake build` first and keeps lines that contain `.lean:line:column`, so a failed candidate proof does not depend on that truncated tail. +- **Envelope:** this section describes the code on this branch. This lane did not re-run the public validation split, and it did not touch the vault or the sealed holdout. diff --git a/src/mcproof/lean_harness.py b/src/mcproof/lean_harness.py index 0c5e74a..f909aa6 100644 --- a/src/mcproof/lean_harness.py +++ b/src/mcproof/lean_harness.py @@ -9,12 +9,14 @@ from pathlib import Path import re import shutil +import stat import subprocess import sys import tempfile -from typing import Any, Sequence +from typing import Any, Mapping, Sequence +import uuid -from .paths import dep_repo, elan_bin_dir, find_lean_tool, repo_root +from .paths import dep_repo, deps_dir, elan_bin_dir, find_lean_tool, repo_root STANDARD_AXIOMS: frozenset[str] = frozenset({"propext", "Classical.choice", "Quot.sound"}) SORRY_WARNING_RE = re.compile(r"warning:.*declaration uses [`']sorry[`']") @@ -59,11 +61,30 @@ ), ), ( + # `attribute [init]` is not `@[init]`. It still runs when the module is imported. + # The marker may sit mid-line or be split across a newline (`attribute\n[init]`). + "FORBIDDEN_INIT_ATTRIBUTE", + re.compile(r"attribute\s*\[[^\]]*\b(?:builtin_init|init)\b"), + ), + ( + # Not only at the start of a line. A mid-line `#eval` still runs during elaboration. "FORBIDDEN_EVAL_COMMAND", - re.compile(r"(?m)^[ \t]*#(?:eval|reduce|check|print|synth)\b"), + re.compile(r"#(?:eval|reduce|check|print|synth)\b"), ), ) +# A hit here means the candidate would run code during compile or import. +# Callers return before any Lean process. Sorry, axiom, and vacuous lints still +# reach the kernel so those checks stay in place. +COMPILE_BLOCKING_LINT_RULES = frozenset({ + "FORBIDDEN_INIT_ATTRIBUTE", + "FORBIDDEN_EVAL_COMMAND", + "METAPROGRAM_ESCAPE", + "SYNTAX_OVERRIDE_DECL", + "UNSOUND_IMPLEMENTED_BY", + "UNVERIFIED_NATIVE_EVAL", +}) + COMPARATOR_ORDERED_KERNELS: tuple[tuple[str, str, str], ...] = ( ("lean-paranoid", "Running Lean paranoid kernel on solution", "Lean paranoid kernel accepts the solution"), ("lean4lean", "Running lean4lean kernel on solution", "lean4lean kernel accepts the solution"), @@ -79,46 +100,252 @@ ) -def linux_bwrap_functional(*, timeout_sec: float = 5.0) -> bool: - """Return True when Linux `bwrap` runs a minimal sandbox and can create `lake comparator`'s `/run/user` session dir.""" +class CandidateSandboxUnavailable(RuntimeError): + """`bwrap` cannot confine a candidate Lean run. Callers must not fall back to the host.""" + + +# Host trees that must not be visible. `/run` hides ssh-agent and API sockets. +# `/agent` and `/workspace` hide the checkout, including `.git/config`. +_SANDBOX_SYS_RO = ("/usr", "/lib", "/lib64", "/bin", "/sbin", "/etc") +_SANDBOX_TMPFS = ("/tmp", "/run", "/home", "/root", "/agent", "/workspace", "/opt", "/var") +_SANDBOX_CPU_SEC = 600 +# `RLIMIT_NPROC` counts every thread of this uid. 8192 is enough for `lake build`. +_SANDBOX_NOFILE = 4096 +_SANDBOX_NPROC = 8192 +_SANDBOX_PROBE_SOCKS = ("/run/cursor/api.sock", "/run/host-services/ssh-auth.sock") +_BWRAP_FUNCTIONAL: bool | None = None + +_SANDBOX_PROBE = """ +import socket, sys +from pathlib import Path +sentinel = sys.argv[1] +git_config = Path(sys.argv[2]) +blob = Path("/proc/1/environ").read_bytes() + b"\\0" + Path("/proc/self/environ").read_bytes() +if sentinel.encode() in blob: + raise SystemExit(2) +if git_config.is_file(): + raise SystemExit(3) +for sock in sys.argv[3:]: + handle = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + handle.settimeout(0.2) + try: + handle.connect(sock) + except OSError: + pass + else: + raise SystemExit(4) + finally: + handle.close() +try: + Path("/etc/mcproof-sandbox-ro-probe").write_text("x", encoding="utf-8") +except OSError: + pass +else: + raise SystemExit(5) +raise SystemExit(0) +""" + + +def _address_space_cap_bytes() -> int: + """Virtual-memory cap for a confined process: this host's `MemTotal`, not more. + + A fixed cap above the machine's RAM does not contain a memory hog. + """ + try: + for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines(): + if line.startswith("MemTotal:"): + kb = int(line.split()[1]) + if kb > 0: + return kb * 1024 + except (OSError, ValueError, IndexError): + pass + return 1024 * 1024 * 1024 + + +def _toolchain_elan_home() -> str: + root = Path(elan_bin_dir()).resolve().parent + return str(root) if (root / "bin").is_dir() else "" + + +def _sandbox_env_pairs(caller_env: Mapping[str, str] | None) -> list[tuple[str, str]]: + """Allowlist passed to `env -i` and to `bwrap --setenv`. Nothing else crosses.""" + elan = _toolchain_elan_home() + path_parts = ["/usr/bin", "/bin"] + if elan: + path_parts.append(str(Path(elan) / "bin")) + pairs = [ + ("PATH", os.pathsep.join(path_parts)), + ("HOME", "/tmp/home"), + ("TMPDIR", "/tmp"), + ("LANG", "C.UTF-8"), + ("LC_ALL", "C.UTF-8"), + ] + if elan: + pairs.append(("ELAN_HOME", elan)) + lean_path = (caller_env or {}).get("LEAN_PATH") + if lean_path: + pairs.append(("LEAN_PATH", lean_path)) + return pairs + + +def _confine_argv( + cmd: Sequence[str], + *, + cwd: Path, + caller_env: Mapping[str, str] | None, + readable_dirs: Sequence[Path], +) -> list[str]: + """`prlimit` + `env -i` + `bwrap` over the toolchain, not the host root.""" + prlimit_bin = shutil.which("prlimit") + env_bin = shutil.which("env") + bwrap_bin = shutil.which("bwrap") + if not prlimit_bin or not env_bin or not bwrap_bin: + raise CandidateSandboxUnavailable( + "Linux bubblewrap (`bwrap`), `env`, or `prlimit` is unavailable; " + "refusing to run candidate Lean unsandboxed." + ) + cwd_path = Path(cwd).resolve() + cwd_s = str(cwd_path) + pairs = _sandbox_env_pairs(caller_env) + argv: list[str] = [ + prlimit_bin, + f"--cpu={_SANDBOX_CPU_SEC}", + f"--as={_address_space_cap_bytes()}", + f"--nofile={_SANDBOX_NOFILE}", + f"--nproc={_SANDBOX_NPROC}", + "--", + env_bin, + "-i", + ] + argv.extend(f"{key}={value}" for key, value in pairs) + argv.extend( + ( + bwrap_bin, + "--new-session", + "--unshare-all", + "--unshare-net", + "--die-with-parent", + "--dev", + "/dev", + "--proc", + "/proc", + ) + ) + for directory in _SANDBOX_SYS_RO: + if Path(directory).is_dir(): + argv.extend(("--ro-bind", directory, directory)) + for directory in _SANDBOX_TMPFS: + if Path(directory).is_dir(): + argv.extend(("--tmpfs", directory)) + # Lake's own sandbox mounts a tmpfs on `/run/user`. The directory has to + # exist; an empty tmpfs over `/run` would make that mkdir fail. + argv.extend(("--dir", "/tmp/home", "--dir", "/run/user")) + elan = _toolchain_elan_home() + if elan: + argv.extend(("--ro-bind", elan, elan)) + deps = deps_dir() + if deps.is_dir(): + deps_s = str(deps.resolve()) + argv.extend(("--ro-bind", deps_s, deps_s)) + for extra in readable_dirs: + extra_path = Path(extra).resolve() + extra_s = str(extra_path) + if not extra_path.exists(): + continue + if extra_s == cwd_s or extra_path.is_relative_to(cwd_path): + continue + argv.extend(("--ro-bind", extra_s, extra_s)) + argv.extend(("--bind", cwd_s, cwd_s, "--clearenv")) + for key, value in pairs: + argv.extend(("--setenv", key, value)) + argv.extend(("--chdir", cwd_s, "--", *cmd)) + return argv + + +def _spawn_confined( + cmd: Sequence[str], + *, + cwd: Path, + caller_env: Mapping[str, str] | None, + timeout_sec: float, + readable_dirs: Sequence[Path] = (), +) -> subprocess.CompletedProcess[str]: + argv = _confine_argv(cmd, cwd=cwd, caller_env=caller_env, readable_dirs=readable_dirs) + return subprocess.run( + argv, + capture_output=True, + text=True, + timeout=timeout_sec, + check=False, + ) + + +def linux_bwrap_functional(*, timeout_sec: float = 15.0) -> bool: + """Return True when the candidate sandbox actually confines a process. + + The probe refuses a parent-env leak in `/proc/1/environ`, a readable + `.git/config`, a connected host socket, or a write on the read-only host. + A missing `bwrap` or `prlimit` is False. Callers must not fall back. + """ + global _BWRAP_FUNCTIONAL + if _BWRAP_FUNCTIONAL is not None: + return _BWRAP_FUNCTIONAL if not sys.platform.startswith("linux"): + _BWRAP_FUNCTIONAL = False return False - bwrap_bin = shutil.which("bwrap") - if not bwrap_bin: + if not shutil.which("bwrap") or not shutil.which("prlimit") or not shutil.which("env"): + _BWRAP_FUNCTIONAL = False return False + sentinel = f"mcproof-sentinel-{uuid.uuid4().hex}" + os.environ["MCPROOF_SANDBOX_SENTINEL"] = sentinel try: - basic = subprocess.run( - [bwrap_bin, "--ro-bind", "/", "/", "true"], - capture_output=True, - timeout=timeout_sec, - check=False, - ) - if basic.returncode != 0: - return False - uid = os.getuid() - lake_compat = subprocess.run( - [ - bwrap_bin, - "--ro-bind", - "/", - "/", - "--bind", - "/run", - "/run", - "--bind", - "/tmp", - "/tmp", - "sh", - "-c", - f"mkdir -p /run/user/{uid} && test -w /run/user/{uid}", - ], - capture_output=True, - timeout=timeout_sec, - check=False, - ) - except (OSError, subprocess.TimeoutExpired): + with tempfile.TemporaryDirectory(prefix="mcproof_bwrap_probe_") as tmp: + script = Path(tmp) / "probe.py" + script.write_text(_SANDBOX_PROBE, encoding="utf-8") + git_config = repo_root() / ".git" / "config" + probe = _spawn_confined( + ["python3", str(script), sentinel, str(git_config), *_SANDBOX_PROBE_SOCKS], + cwd=Path(tmp), + caller_env={}, + timeout_sec=timeout_sec, + ) + except (OSError, subprocess.TimeoutExpired, CandidateSandboxUnavailable): + _BWRAP_FUNCTIONAL = False return False - return lake_compat.returncode == 0 + finally: + os.environ.pop("MCPROOF_SANDBOX_SENTINEL", None) + _BWRAP_FUNCTIONAL = probe.returncode == 0 + return _BWRAP_FUNCTIONAL + + +def run_confined_command( + cmd: Sequence[str], + *, + cwd: Path, + env: Mapping[str, str] | None = None, + timeout_sec: float = 300, + readable_dirs: Sequence[Path] = (), +) -> subprocess.CompletedProcess[str]: + """Run `cmd` where the candidate cannot see the host. + + `prlimit` caps cpu, address space, files, and processes. `env -i` and + `--new-session` start `bwrap` with only the toolchain allowlist, so + `/proc/1/environ` is not the parent environment. The mount is the toolchain, + system libraries, and `deps`, with `tmpfs` over `/run`, `/agent`, and + `/workspace`. Only `cwd` is writable. A missing or non-functional sandbox + raises `CandidateSandboxUnavailable` and never runs `cmd` on the host. + """ + if not linux_bwrap_functional(): + raise CandidateSandboxUnavailable( + "Linux bubblewrap (`bwrap`) is unavailable; refusing to run candidate Lean unsandboxed." + ) + return _spawn_confined( + cmd, + cwd=cwd, + caller_env=env, + timeout_sec=timeout_sec, + readable_dirs=readable_dirs, + ) AXIOM_AUDIT_TEMPLATE = """import Lean @@ -185,8 +412,144 @@ def _load_soko_lean4_auditor() -> Any: return mod +def _is_lean_letter_like(ch: str) -> bool: + """Lean `isLetterLike` ranges used to keep identifier tokens intact.""" + o = ord(ch) + if 0x3B1 <= o <= 0x3C9 and o != 0x3BB: + return True + if 0x391 <= o <= 0x3A9 and o not in (0x3A0, 0x3A3): + return True + if 0x3CA <= o <= 0x3FB or 0x1F00 <= o <= 0x1FFE: + return True + if 0x2100 <= o <= 0x214F or 0x1D49C <= o <= 0x1D59F: + return True + if 0x00C0 <= o <= 0x00FF and o not in (0x00D7, 0x00F7): + return True + return 0x0100 <= o <= 0x017F + + +def _is_lean_subscript_alnum(ch: str) -> bool: + o = ord(ch) + return ( + 0x2080 <= o <= 0x2089 + or 0x2090 <= o <= 0x209C + or 0x1D62 <= o <= 0x1D6A + or o == 0x2C7C + ) + + +def _is_lean_id_first(ch: str) -> bool: + return ch.isalpha() or ch == "_" or _is_lean_letter_like(ch) + + +def _is_lean_id_rest(ch: str) -> bool: + return ( + ch.isalnum() + or ch in "_'!?" + or _is_lean_letter_like(ch) + or _is_lean_subscript_alnum(ch) + ) + + +def _is_hex_digit(ch: str) -> bool: + return ch.isdigit() or "a" <= ch <= "f" or "A" <= ch <= "F" + + +def _lean_ident_end(src: str, i: int) -> int: + """End index of the identifier that starts at `i`, including dotted segments and `«...»` parts.""" + n = len(src) + while i < n: + if src[i] == "«": + j = src.find("»", i + 1) + i = n if j < 0 else j + 1 + elif _is_lean_id_first(src[i]): + i += 1 + while i < n and _is_lean_id_rest(src[i]): + i += 1 + else: + break + if i < n and src[i] == "." and i + 1 < n and (src[i + 1] == "«" or _is_lean_id_first(src[i + 1])): + i += 1 + continue + break + return i + + +def _is_raw_str_start(src: str, i: int) -> bool: + """True when `src[i]` is `r` and the token is `r` plus zero or more `#` plus `"`.""" + if i >= len(src) or src[i] != "r": + return False + j = i + 1 + while j < len(src) and src[j] == "#": + j += 1 + return j < len(src) and src[j] == '"' + + +def _append_raw_string(src: str, i: int, out: list[str]) -> int: + """Blank a Lean raw string (`r"..."`, `r#"..."#`, any hash count). `\\` is not an escape.""" + n = len(src) + j = i + 1 + hashes = 0 + while j < n and src[j] == "#": + hashes += 1 + j += 1 + if j >= n or src[j] != '"': + out.append(src[i]) + return i + 1 + j += 1 + out.append('"') + while j < n: + if src[j] == '"': + k = j + 1 + seen = 0 + while seen < hashes and k < n and src[k] == "#": + seen += 1 + k += 1 + if seen == hashes: + j = k + break + if src[j] == "\n": + out.append("\n") + j += 1 + out.append('"') + return j + + +def _char_lit_end(src: str, i: int) -> int | None: + """End index of a Lean character literal at `i`, or None when `src[i:]` is not one. + + Covers `'a'`, `'"'`, and escapes (`'\\n'`, `'\\''`, `'\\"'`, `'\\\\'`, `\\xNN`, `\\uNNNN`). + `''` is not a character literal. + """ + n = len(src) + if i >= n or src[i] != "'" or i + 1 >= n or src[i + 1] == "'": + return None + j = i + 1 + if src[j] == "\\": + if j + 1 >= n: + return None + nxt = src[j + 1] + if nxt in "\\\"'rnt": + j += 2 + elif nxt == "x" and j + 3 < n and _is_hex_digit(src[j + 2]) and _is_hex_digit(src[j + 3]): + j += 4 + elif nxt == "u" and j + 5 < n and all(_is_hex_digit(src[j + k]) for k in range(2, 6)): + j += 6 + else: + return None + else: + j += 1 + if j < n and src[j] == "'": + return j + 1 + return None + + def strip_lean_comments_and_strings(src: str) -> str: - """Strip `--` line comments, nested `/- ... -/` block comments, and string literals while keeping line numbers.""" + """Strip `--` line comments, nested `/- ... -/` block comments, and string literals while keeping line numbers. + + Raw strings (`r"..."`, `r#"..."#`, any number of hashes) do not treat `\\` as an escape. + Character literals, including `'"'` and escaped forms, are kept so a following command stays visible. + """ out: list[str] = [] i = 0 n = len(src) @@ -210,6 +573,20 @@ def strip_lean_comments_and_strings(src: str) -> str: j = src.find("\n", i) i = n if j < 0 else j continue + if _is_raw_str_start(src, i): + i = _append_raw_string(src, i, out) + continue + if src[i] == "«" or _is_lean_id_first(src[i]): + j = _lean_ident_end(src, i) + out.append(src[i:j]) + i = j + continue + if src[i] == "'": + end = _char_lit_end(src, i) + if end is not None: + out.append(src[i:end]) + i = end + continue if src[i] == '"': i += 1 out.append('"') @@ -660,6 +1037,14 @@ def verify_lean4_prompt_obligations( if findings or not run_kernel_type_check or not obligations["theorem_specs"]: return findings + for item in lint_lean4_source( + candidate_source, require_theorem=False, check_vacuous=False, + )["findings"]: + if item.get("rule") in COMPILE_BLOCKING_LINT_RULES: + findings.append(item) + if findings: + return findings + proj_dir = (lean_project_dir or (repo_root() / "lean")).resolve() lean_bin = find_lean_tool("lean") env = _lean_env() @@ -715,27 +1100,27 @@ def verify_lean4_prompt_obligations( run_env = dict(env) run_env["LEAN_PATH"] = os.pathsep.join(lean_path_parts) - c_proc = subprocess.run( - [lean_bin, "-j", "1", "-D", "Elab.async=false", str(cand_f), "-o", str(cand_o)], - cwd=tdp, - env=run_env, - capture_output=True, - text=True, - timeout=timeout_sec, - check=False, - ) - if c_proc.returncode != 0: + readable = (lib_olean_dir,) if lib_olean_dir.is_dir() else () + try: + c_proc = run_confined_command( + [lean_bin, "-j", "1", "-D", "Elab.async=false", str(cand_f), "-o", str(cand_o)], + cwd=tdp, + env=run_env, + timeout_sec=timeout_sec, + readable_dirs=readable, + ) + if c_proc.returncode != 0: + return findings + d_proc = run_confined_command( + [lean_bin, "-j", "1", "-D", "Elab.async=false", str(drv_f)], + cwd=tdp, + env=run_env, + timeout_sec=timeout_sec, + readable_dirs=readable, + ) + except CandidateSandboxUnavailable as exc: + findings.append({"line": 1, "rule": "SANDBOX_UNAVAILABLE", "message": str(exc)}) return findings - - d_proc = subprocess.run( - [lean_bin, "-j", "1", "-D", "Elab.async=false", str(drv_f)], - cwd=tdp, - env=run_env, - capture_output=True, - text=True, - timeout=timeout_sec, - check=False, - ) if d_proc.returncode != 0: tail = " | ".join(ln.strip() for ln in ((d_proc.stdout or "") + "\n" + (d_proc.stderr or "")).splitlines()[-4:] if ln.strip()) findings.append({ @@ -765,11 +1150,19 @@ def verify_lean4_against_challenge( timeout_sec: float = 60.0, ) -> list[dict[str, Any]]: """Verify via Lean `Meta.isDefEq` that `candidate_source` implements every def/inductive/theorem in `challenge_lean`.""" + findings: list[dict[str, Any]] = [ + item for item in lint_lean4_source( + candidate_source, require_theorem=False, check_vacuous=False, + )["findings"] + if item.get("rule") in COMPILE_BLOCKING_LINT_RULES + ] + if findings: + return findings + proj_dir = (lean_project_dir or (repo_root() / "lean")).resolve() lean_bin = find_lean_tool("lean") env = _lean_env() lib_olean_dir = proj_dir / ".lake" / "build" / "lib" / "lean" - findings: list[dict[str, Any]] = [] with tempfile.TemporaryDirectory(prefix="mcproof_chalcmp_") as td: tdp = Path(td).resolve() @@ -792,41 +1185,39 @@ def verify_lean4_against_challenge( run_env = dict(env) run_env["LEAN_PATH"] = os.pathsep.join(lean_path_parts) - s_proc = subprocess.run( - [lean_bin, "-j", "1", "-D", "Elab.async=false", str(spec_f), "-o", str(spec_o)], - cwd=tdp, - env=run_env, - capture_output=True, - text=True, - timeout=timeout_sec, - check=False, - ) - c_proc = subprocess.run( - [lean_bin, "-j", "1", "-D", "Elab.async=false", str(cand_f), "-o", str(cand_o)], - cwd=tdp, - env=run_env, - capture_output=True, - text=True, - timeout=timeout_sec, - check=False, - ) - if s_proc.returncode != 0 or c_proc.returncode != 0: - findings.append({ - "line": 1, - "rule": "CHALLENGE_COMPARE_BUILD_FAILED", - "message": "Failed to compile challenge or candidate module for kernel statement comparison.", - }) + readable = (lib_olean_dir,) if lib_olean_dir.is_dir() else () + try: + s_proc = run_confined_command( + [lean_bin, "-j", "1", "-D", "Elab.async=false", str(spec_f), "-o", str(spec_o)], + cwd=tdp, + env=run_env, + timeout_sec=timeout_sec, + readable_dirs=readable, + ) + c_proc = run_confined_command( + [lean_bin, "-j", "1", "-D", "Elab.async=false", str(cand_f), "-o", str(cand_o)], + cwd=tdp, + env=run_env, + timeout_sec=timeout_sec, + readable_dirs=readable, + ) + if s_proc.returncode != 0 or c_proc.returncode != 0: + findings.append({ + "line": 1, + "rule": "CHALLENGE_COMPARE_BUILD_FAILED", + "message": "Failed to compile challenge or candidate module for kernel statement comparison.", + }) + return findings + cmp_proc = run_confined_command( + [lean_bin, "-j", "1", "-D", "Elab.async=false", str(drv_f)], + cwd=tdp, + env=run_env, + timeout_sec=timeout_sec, + readable_dirs=readable, + ) + except CandidateSandboxUnavailable as exc: + findings.append({"line": 1, "rule": "SANDBOX_UNAVAILABLE", "message": str(exc)}) return findings - - cmp_proc = subprocess.run( - [lean_bin, "-j", "1", "-D", "Elab.async=false", str(drv_f)], - cwd=tdp, - env=run_env, - capture_output=True, - text=True, - timeout=timeout_sec, - check=False, - ) if cmp_proc.returncode != 0: tail = " | ".join(ln.strip() for ln in ((cmp_proc.stdout or "") + "\n" + (cmp_proc.stderr or "")).splitlines()[-4:] if ln.strip()) findings.append({ @@ -1083,23 +1474,71 @@ def verify_lean4_source( require_theorem=require_theorem, ) findings: list[dict[str, Any]] = list(lint["findings"]) + # An execution lint means the source would run during compile or import. + # Returning here is what stops the host write. The sandbox is the other half. + if any(item.get("rule") in COMPILE_BLOCKING_LINT_RULES for item in findings): + return LeanVerificationResult( + passed=False, + toolchain=toolchain, + theorems_checked=(), + axioms_used=(), + nonstandard_axioms=(), + sorry_declarations=(), + paranoid_kernels_run=(), + sandboxed=False, + findings=tuple(findings), + ) lean_bin = find_lean_tool("lean") lake_bin = find_lean_tool("lake") leanchecker_bin = find_lean_tool("leanchecker") env = _lean_env() - # Ensure reference Lean library `.olean` files exist if project not yet built + # Ensure reference Lean library `.olean` files exist if project not yet built. + # The build is confined too: this function does not run Lean on the host. lib_olean_dir = proj_dir / ".lake" / "build" / "lib" / "lean" - if not lib_olean_dir.is_dir(): - subprocess.run( - [lake_bin, "build", "Mcproof"], - cwd=proj_dir, - env=env, - capture_output=True, - text=True, - timeout=timeout_sec, - check=False, + try: + if not lib_olean_dir.is_dir(): + run_confined_command( + [lake_bin, "build", "Mcproof"], + cwd=proj_dir, + env=env, + timeout_sec=timeout_sec, + ) + except CandidateSandboxUnavailable as exc: + findings.append({ + "line": 1, + "rule": "SANDBOX_UNAVAILABLE", + "message": str(exc), + }) + return LeanVerificationResult( + passed=False, + toolchain=toolchain, + theorems_checked=(), + axioms_used=(), + nonstandard_axioms=(), + sorry_declarations=(), + paranoid_kernels_run=(), + sandboxed=False, + findings=tuple(findings), + ) + except subprocess.TimeoutExpired as exc: + findings.append({ + "line": 1, + "rule": "HOST_RESOURCE_EXHAUSTED", + "message": f"Lean elaboration timed out after {timeout_sec}s ({exc})", + }) + return LeanVerificationResult( + passed=False, + toolchain=toolchain, + theorems_checked=(), + axioms_used=(), + nonstandard_axioms=(), + sorry_declarations=(), + paranoid_kernels_run=(), + sandboxed=False, + findings=tuple(findings), + host_resource_exhausted=True, ) mod_name = "McproofCandidateCheck" @@ -1128,14 +1567,12 @@ def verify_lean4_source( max_memory_mib=max_memory_mib, ) try: - comp = subprocess.run( + comp = run_confined_command( comp_cmd, cwd=tmp_dir, env=run_env, - capture_output=True, - text=True, - timeout=timeout_sec, - check=False, + timeout_sec=timeout_sec, + readable_dirs=(lib_olean_dir,) if lib_olean_dir.is_dir() else (), ) except subprocess.TimeoutExpired as exc: findings.append({ @@ -1151,10 +1588,23 @@ def verify_lean4_source( nonstandard_axioms=(), sorry_declarations=(), paranoid_kernels_run=(), - sandboxed=False, + sandboxed=True, findings=tuple(findings), host_resource_exhausted=True, ) + except CandidateSandboxUnavailable as exc: + findings.append({"line": 1, "rule": "SANDBOX_UNAVAILABLE", "message": str(exc)}) + return LeanVerificationResult( + passed=False, + toolchain=toolchain, + theorems_checked=(), + axioms_used=(), + nonstandard_axioms=(), + sorry_declarations=(), + paranoid_kernels_run=(), + sandboxed=False, + findings=tuple(findings), + ) combined_out = (comp.stdout or "") + "\n" + (comp.stderr or "") if comp.returncode != 0: host_reason = classify_host_resource_failure(comp.returncode, combined_out) @@ -1172,7 +1622,7 @@ def verify_lean4_source( nonstandard_axioms=(), sorry_declarations=(), paranoid_kernels_run=(), - sandboxed=False, + sandboxed=True, findings=tuple(findings), host_resource_exhausted=True, ) @@ -1190,7 +1640,7 @@ def verify_lean4_source( nonstandard_axioms=(), sorry_declarations=(), paranoid_kernels_run=(), - sandboxed=False, + sandboxed=True, findings=tuple(findings), ) @@ -1208,15 +1658,27 @@ def verify_lean4_source( AXIOM_AUDIT_TEMPLATE.format(module_name=mod_name, root_json=json.dumps(mod_name)), encoding="utf-8", ) - ax_run = subprocess.run( - [lean_bin, str(audit_driver)], - cwd=tmp_dir, - env=run_env, - capture_output=True, - text=True, - timeout=timeout_sec, - check=False, - ) + try: + ax_run = run_confined_command( + [lean_bin, str(audit_driver)], + cwd=tmp_dir, + env=run_env, + timeout_sec=timeout_sec, + readable_dirs=(lib_olean_dir,) if lib_olean_dir.is_dir() else (), + ) + except CandidateSandboxUnavailable as exc: + findings.append({"line": 1, "rule": "SANDBOX_UNAVAILABLE", "message": str(exc)}) + return LeanVerificationResult( + passed=False, + toolchain=toolchain, + theorems_checked=(), + axioms_used=(), + nonstandard_axioms=(), + sorry_declarations=(), + paranoid_kernels_run=tuple(kernels_run), + sandboxed=False, + findings=tuple(findings), + ) if ax_run.returncode != 0: tail = " | ".join(line.strip() for line in ((ax_run.stdout or "") + "\n" + (ax_run.stderr or "")).splitlines()[-4:] if line.strip()) findings.append({ @@ -1266,15 +1728,27 @@ def verify_lean4_source( }) if run_paranoid_leanchecker and cand_olean.is_file(): - lc_run = subprocess.run( - [leanchecker_bin, mod_name], - cwd=tmp_dir, - env=run_env, - capture_output=True, - text=True, - timeout=timeout_sec, - check=False, - ) + try: + lc_run = run_confined_command( + [leanchecker_bin, mod_name], + cwd=tmp_dir, + env=run_env, + timeout_sec=timeout_sec, + readable_dirs=(lib_olean_dir,) if lib_olean_dir.is_dir() else (), + ) + except CandidateSandboxUnavailable as exc: + findings.append({"line": 1, "rule": "SANDBOX_UNAVAILABLE", "message": str(exc)}) + return LeanVerificationResult( + passed=False, + toolchain=toolchain, + theorems_checked=tuple(sorted(theorems_checked)), + axioms_used=tuple(sorted(all_axioms)), + nonstandard_axioms=tuple(sorted(nonstandard)), + sorry_declarations=tuple(sorted(sorry_decls)), + paranoid_kernels_run=tuple(kernels_run), + sandboxed=False, + findings=tuple(findings), + ) if lc_run.returncode != 0: tail = " | ".join(line.strip() for line in ((lc_run.stdout or "") + "\n" + (lc_run.stderr or "")).splitlines()[-4:] if line.strip()) findings.append({ @@ -1299,11 +1773,437 @@ def verify_lean4_source( nonstandard_axioms=tuple(sorted(nonstandard)), sorry_declarations=tuple(sorted(sorry_decls)), paranoid_kernels_run=tuple(kernels_run), - sandboxed=False, + sandboxed=True, findings=tuple(findings), ) +class UntrustedProjectPath(Exception): + """A copied path is a symlink or a special file. The trusted copy refuses it.""" + + def __init__(self, message: str, *, rule: str = "UNTRUSTED_PROJECT_SYMLINK") -> None: + super().__init__(message) + self.rule = rule + + +# `lstat(dir_fd=)` is the `stat` implementation. CPython lists `stat` in +# `os.supports_dir_fd` and does not list `lstat` itself. +_NOFOLLOW_DIR_OPS = ("open", "stat", "mkdir") +_NOFOLLOW_OPEN_RULE = "NOFOLLOW_OPEN_UNAVAILABLE" + + +def _nofollow_dir_ops_available() -> bool: + """True when a directory can be opened without following a symlink in any component.""" + if not hasattr(os, "O_DIRECTORY") or not hasattr(os, "O_NOFOLLOW"): + return False + supported = os.supports_dir_fd + return all(getattr(os, name) in supported for name in _NOFOLLOW_DIR_OPS) + + +def _nofollow_unavailable_finding() -> dict[str, Any]: + return { + "line": 1, + "rule": _NOFOLLOW_OPEN_RULE, + "message": ( + "O_DIRECTORY, O_NOFOLLOW, or dir_fd is missing; " + "refusing to open the candidate project because a following copy is not safe." + ), + } + + +def _refuse_nofollow_unavailable() -> None: + if _nofollow_dir_ops_available(): + return + finding = _nofollow_unavailable_finding() + raise UntrustedProjectPath(str(finding["message"]), rule=_NOFOLLOW_OPEN_RULE) + + +def finding_blocks_comparator(finding: Mapping[str, Any]) -> bool: + """An elaboration command means the comparator must not run. + + `#eval`, `run_cmd`, and `run_meta` plant files during the build. The + comparator elaborates the project, so it must not start once one of them + has been seen. `elab_rules` stays out of this set: the confined build is + what contains it. + """ + rule = str(finding.get("rule", "")) + message = str(finding.get("message", "")) + if rule in {"FORBIDDEN_EVAL_COMMAND", "FORBIDDEN_ELAB_COMMAND"}: + return True + tokens = ("run_meta", "run_cmd", "run_elab", "run_tac", "#eval", "#reduce", "#check", "#print", "#synth") + return rule in {"METAPROGRAM_ESCAPE", "FORBIDDEN_BUILD_SIDE_EFFECT"} and any(token in message for token in tokens) + + +_LAKE_COMMAND_RE = re.compile( + r"(? tuple[int, int | None]: + """End of a terminated raw string, or where a short closer resumes code. + + Zero hashes close at the next `"`. A positive hash count closes only at `"` + plus that many hashes. A `"` with fewer hashes is the error Lean recovers + from by elaborating the following command, which the comment stripper still + hides when the closer is short. + """ + n = len(src) + j = i + 1 + hashes = 0 + while j < n and src[j] == "#": + hashes += 1 + j += 1 + if j >= n or src[j] != '"': + return i + 1, None + j += 1 + if hashes == 0: + while j < n and src[j] != '"': + j += 1 + return (j + 1, None) if j < n else (n, None) + while j < n: + if src[j] == '"': + k = j + 1 + seen = 0 + while seen < hashes and k < n and src[k] == "#": + seen += 1 + k += 1 + if seen == hashes: + return k, None + return n, k + j += 1 + return n, None + + +def _lean_commands_that_may_run(src: str) -> list[str]: + """Commands Lean can elaborate, including one hidden by a short raw-string closer. + + A command inside a terminated string or comment is not included. The comment + stripper is unchanged, so a compile that must still run those payloads is + unchanged too. + """ + found: list[str] = [] + n = len(src) + i = 0 + while i < n: + if src.startswith("/-", i): + depth = 1 + i += 2 + while i < n and depth > 0: + if src.startswith("/-", i): + depth += 1 + i += 2 + elif src.startswith("-/", i): + depth -= 1 + i += 2 + else: + i += 1 + continue + if src.startswith("--", i): + newline = src.find("\n", i) + i = n if newline < 0 else newline + continue + if _is_raw_str_start(src, i): + end, resume = _raw_string_resume(src, i) + if resume is not None and resume > i: + i = resume + else: + i = end if end > i else i + 1 + continue + match = _LAKE_COMMAND_RE.match(src, i) + if match: + cmd = match.group(0) + if cmd not in found: + found.append(cmd) + i += len(cmd) + continue + if src[i] == '"': + i += 1 + while i < n and src[i] != '"': + if src[i] == "\\" and i + 1 < n: + i += 2 + else: + i += 1 + if i < n: + i += 1 + continue + if src[i] == "'": + end = _char_lit_end(src, i) + if end is not None: + i = end + continue + if src[i] == "«" or _is_lean_id_first(src[i]): + i = _lean_ident_end(src, i) + continue + i += 1 + return found + + +def _command_findings(rel: str, text: str) -> list[dict[str, Any]]: + findings: list[dict[str, Any]] = [] + for cmd in _lean_commands_that_may_run(text): + if cmd.startswith("#"): + findings.append({ + "line": 1, + "rule": "FORBIDDEN_EVAL_COMMAND", + "message": f"{rel} contains a Lean command that runs during elaboration: {cmd}", + }) + else: + findings.append({ + "line": 1, + "rule": "METAPROGRAM_ESCAPE", + "message": f"forbidden Lean escape hatch matched {cmd!r}", + }) + return findings + + +def _lake_symlink_finding(proj: Path) -> dict[str, Any] | None: + """`.lake` as a symlink is a finding. `rmtree` on that path raises, and following it deletes the target.""" + if not _nofollow_dir_ops_available(): + return _nofollow_unavailable_finding() + try: + root_fd = os.open(proj, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + except OSError as exc: + return { + "line": 1, + "rule": "UNTRUSTED_PROJECT_SYMLINK", + "message": f"project root is not a real directory: {exc}", + } + try: + try: + st = os.lstat(".lake", dir_fd=root_fd) + except FileNotFoundError: + return None + if stat.S_ISLNK(st.st_mode): + return { + "line": 1, + "rule": "LAKE_CACHE_SYMLINK", + "message": ".lake is a symlink; refusing to follow or delete it", + } + return None + finally: + os.close(root_fd) + + +def _read_project_tree(proj: Path) -> dict[str, bytes]: + """Regular files only. A symlink or special file in any component refuses the copy. + + `.lake` and `.git` are not copied. Opening each component with `O_NOFOLLOW` + does not follow a parent the candidate swapped for a symlink. + """ + _refuse_nofollow_unavailable() + root_fd = os.open(proj, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + files: dict[str, bytes] = {} + _read_tree_at(root_fd, "", files) + return files + finally: + os.close(root_fd) + + +def _read_tree_at(dir_fd: int, rel: str, files: dict[str, bytes]) -> None: + for name in sorted(os.listdir(f"/proc/self/fd/{dir_fd}")): + if name in {".lake", ".git"}: + continue + child = f"{rel}/{name}" if rel else name + st = os.lstat(name, dir_fd=dir_fd) + if stat.S_ISLNK(st.st_mode): + raise UntrustedProjectPath(f"{child} is a symlink") + if stat.S_ISDIR(st.st_mode): + child_fd = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=dir_fd) + try: + _read_tree_at(child_fd, child, files) + finally: + os.close(child_fd) + continue + if not stat.S_ISREG(st.st_mode): + raise UntrustedProjectPath(f"{child} is not a regular file", rule="UNTRUSTED_PROJECT_FILE") + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=dir_fd) + try: + chunks: list[bytes] = [] + while True: + block = os.read(fd, 1024 * 1024) + if not block: + break + chunks.append(block) + files[child] = b"".join(chunks) + finally: + os.close(fd) + + +def _trusted_inputs(files: dict[str, bytes]) -> dict[str, bytes]: + """Drop a lakefile Lean would prefer over `lakefile.toml`. + + The toml is the trusted configuration. `lakefile.lean` beside it is the candidate's plant. + A `lake-manifest.json` that is already here is trusted. One the build creates later is not + in this map, so a fresh tree does not receive it. + """ + if "lakefile.toml" not in files: + return dict(files) + kept: dict[str, bytes] = {} + for rel, data in files.items(): + parent, _, name = rel.rpartition("/") + if parent == "" and name.startswith("lakefile.") and name != "lakefile.toml": + continue + kept[rel] = data + return kept + + +def _materialize_trusted_tree(files: dict[str, bytes]) -> Path: + """A new directory of regular files. Nothing here is a path the candidate chose.""" + _refuse_nofollow_unavailable() + root = Path(tempfile.mkdtemp(prefix="mcproof_trusted_")) + root_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + for rel in sorted(files): + _write_regular_at(root_fd, rel, files[rel]) + except Exception: + os.close(root_fd) + shutil.rmtree(root) + raise + os.close(root_fd) + return root + + +def _write_regular_at(root_fd: int, rel: str, data: bytes) -> None: + parts = rel.split("/") + dir_fd = root_fd + opened: list[int] = [] + try: + for part in parts[:-1]: + try: + os.mkdir(part, dir_fd=dir_fd) + except FileExistsError: + pass + st = os.lstat(part, dir_fd=dir_fd) + if stat.S_ISLNK(st.st_mode) or not stat.S_ISDIR(st.st_mode): + raise UntrustedProjectPath(f"{rel} crosses a symlink") + next_fd = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=dir_fd) + opened.append(next_fd) + dir_fd = next_fd + fd = os.open( + parts[-1], + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o644, + dir_fd=dir_fd, + ) + try: + view = memoryview(data) + while view: + wrote = os.write(fd, view) + view = view[wrote:] + finally: + os.close(fd) + finally: + for fd in reversed(opened): + os.close(fd) + + +def _real_dir(parent: Path, rel: str) -> bool: + """True when every component of `parent/rel` is a real directory.""" + if not _nofollow_dir_ops_available(): + return False + try: + dir_fd = os.open(parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + except OSError: + return False + try: + for part in rel.split("/"): + try: + st = os.lstat(part, dir_fd=dir_fd) + except FileNotFoundError: + return False + if stat.S_ISLNK(st.st_mode) or not stat.S_ISDIR(st.st_mode): + return False + next_fd = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=dir_fd) + os.close(dir_fd) + dir_fd = next_fd + return True + finally: + os.close(dir_fd) + + +def _audit_solution_axioms( + build_dir: Path, + *, + solution_mod: str, + permitted: set[str], + lean_bin: str, + env: dict[str, str], + timeout_sec: int, + findings: list[dict[str, Any]], +) -> tuple[set[str], list[str], list[str], str | None]: + """Import `solution_mod` from the build directory. The last item is a sandbox refusal, if any.""" + measured: set[str] = set() + sorry_decls: list[str] = [] + nonstd: list[str] = [] + with tempfile.TemporaryDirectory(prefix="mcproof_proj_ax_") as ax_tmp_str: + ax_tmp = Path(ax_tmp_str) + audit_driver = ax_tmp / "McproofProjectAxiomDriver.lean" + audit_driver.write_text( + MODULE_AXIOM_AUDIT_TEMPLATE.format(module_name=solution_mod), + encoding="utf-8", + ) + ax_env = dict(env) + lean_path_parts = [str(ax_tmp)] + olean_rel = ".lake/build/lib/lean" + if _real_dir(build_dir, olean_rel): + lean_path_parts.append(str(build_dir / olean_rel)) + ax_env["LEAN_PATH"] = os.pathsep.join(lean_path_parts) + try: + ax_run = run_confined_command( + [lean_bin, str(audit_driver)], + cwd=ax_tmp, + env=ax_env, + timeout_sec=timeout_sec, + readable_dirs=(build_dir,), + ) + except CandidateSandboxUnavailable as exc: + return measured, sorry_decls, nonstd, str(exc) + if ax_run.returncode != 0: + tail = " | ".join( + line.strip() + for line in ((ax_run.stdout or "") + "\n" + (ax_run.stderr or "")).splitlines()[-4:] + if line.strip() + ) + findings.append({ + "line": 1, + "rule": "AXIOM_WALK_FAILED", + "message": f"trusted collectAxioms driver failed on {solution_mod}: {tail}", + }) + return measured, sorry_decls, nonstd, None + for line in (ax_run.stdout or "").splitlines(): + if not line.startswith("MCPROOF_AUDIT\t"): + continue + parts = line.split("\t") + if len(parts) != 5: + continue + _, kind, _, decl_name, ax_csv = parts + axs = {a for a in ax_csv.split(",") if a} + measured.update(axs) + if kind in ("axiom", "opaque"): + findings.append({ + "line": 1, + "rule": "KERNEL_EXTRA_AXIOM_OR_OPAQUE", + "message": f"solution module {solution_mod} contains {kind} declaration '{decl_name}'", + }) + if "sorryAx" in axs: + sorry_decls.append(decl_name) + findings.append({ + "line": 1, + "rule": "KERNEL_SORRY_AXIOM", + "message": f"solution declaration '{decl_name}' depends on 'sorryAx'", + }) + extra_axs = axs - (permitted & STANDARD_AXIOMS) - {"sorryAx"} + if extra_axs: + nonstd.extend(sorted(extra_axs)) + findings.append({ + "line": 1, + "rule": "KERNEL_NONSTANDARD_AXIOM", + "message": f"solution declaration '{decl_name}' uses unpermitted axioms: {sorted(extra_axs)}", + }) + return measured, sorry_decls, nonstd, None + + def verify_lean4_project_comparator( project_dir: Path | None = None, *, @@ -1311,16 +2211,80 @@ def verify_lean4_project_comparator( allow_unsandboxed: bool = False, timeout_sec: int = 300, ) -> LeanVerificationResult: - """Run full 6-kernel `lake comparator --paranoid` + `collectAxioms` + source escape-hatch checks on a Lake project.""" + """Run full 6-kernel `lake comparator --paranoid` + `collectAxioms` + source escape-hatch checks on a Lake project. + + Trusted steps run in a directory built from an in-memory copy. The caller's tree is never + written. `allow_unsandboxed` does not run candidate Lean on the host. + """ proj = (project_dir or (repo_root() / "lean")).resolve() - toolchain = _read_toolchain(proj) - lean_bin = find_lean_tool("lean") - lake_bin = find_lean_tool("lake") - env = _lean_env() findings: list[dict[str, Any]] = [] - cmp_path = proj / "comparator.json" - if not cmp_path.is_file(): + def _stopped(*, sandboxed: bool = False, caveats: tuple[str, ...] = ()) -> LeanVerificationResult: + return LeanVerificationResult( + passed=False, + toolchain=toolchain, + theorems_checked=theorem_names, + axioms_used=(), + nonstandard_axioms=(), + sorry_declarations=(), + paranoid_kernels_run=(), + sandboxed=sandboxed, + findings=tuple(findings), + caveats=caveats, + challenge_sorry_whitelisted_module=challenge_mod or None, + ) + + lake_hit = _lake_symlink_finding(proj) + if lake_hit is not None: + # The platform refusal must not stat or read through a candidate path. + if lake_hit.get("rule") == _NOFOLLOW_OPEN_RULE: + toolchain = "leanprover/lean4:v4.35.0" + else: + toolchain = _read_toolchain(proj) if (proj / "lean-toolchain").is_file() and not (proj / "lean-toolchain").is_symlink() else "leanprover/lean4:v4.35.0" + theorem_names = () + challenge_mod = "" + findings.append(lake_hit) + return _stopped() + + try: + project_files = _read_project_tree(proj) + except UntrustedProjectPath as exc: + toolchain = "leanprover/lean4:v4.35.0" + theorem_names = () + challenge_mod = "" + findings.append({"line": 1, "rule": exc.rule, "message": str(exc)}) + return _stopped() + except OSError as exc: + toolchain = "leanprover/lean4:v4.35.0" + theorem_names = () + challenge_mod = "" + findings.append({ + "line": 1, + "rule": "UNTRUSTED_PROJECT_SYMLINK", + "message": f"project root is not a real directory: {exc}", + }) + return _stopped() + + toolchain = project_files.get("lean-toolchain", b"").decode("utf-8", "replace").strip() or "leanprover/lean4:v4.35.0" + raw_cmp = project_files.get("comparator.json") + if raw_cmp is None: + theorem_names = () + challenge_mod = "" + findings.append({"line": 1, "rule": "MISSING_COMPARATOR_JSON", "message": f"no comparator.json in {proj}"}) + return _stopped() + + try: + cmp_cfg = json.loads(raw_cmp.decode("utf-8")) + except json.JSONDecodeError as exc: + theorem_names = () + challenge_mod = "" + findings.append({ + "line": 1, + "rule": "COMPARATOR_JSON_MALFORMED", + "message": f"comparator.json is not valid JSON: {exc}", + }) + return _stopped() + if not isinstance(cmp_cfg, dict): return LeanVerificationResult( passed=False, toolchain=toolchain, @@ -1330,10 +2294,12 @@ def verify_lean4_project_comparator( sorry_declarations=(), paranoid_kernels_run=(), sandboxed=False, - findings=({"line": 1, "rule": "MISSING_COMPARATOR_JSON", "message": f"no comparator.json in {proj}"},), + findings=({ + "line": 1, + "rule": "COMPARATOR_JSON_MALFORMED", + "message": "comparator.json must be a JSON object", + },), ) - - cmp_cfg = json.loads(cmp_path.read_text(encoding="utf-8")) challenge_mod = str(cmp_cfg.get("challenge_module", "")) solution_mod = str(cmp_cfg.get("solution_module", "Mcproof.Proofs")) if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*", solution_mod): @@ -1359,199 +2325,174 @@ def verify_lean4_project_comparator( "message": f"comparator.json permits non-standard axioms: {sorted(extra_permitted)}", }) - # Fail closed before compiling untrusted Lean code when Linux bubblewrap is unavailable unless explicitly opted in - bwrap_available = linux_bwrap_functional() - caveats: list[str] = [] - if not bwrap_available: - if not allow_unsandboxed: - findings.append({ - "line": 1, - "rule": "SANDBOX_UNAVAILABLE", - "message": ( - "Linux bubblewrap (`bwrap`) is unavailable on this host; " - "pass allow_unsandboxed=True or `--allow-unsandboxed` to explicitly authorize unsandboxed execution" - ), - }) - return LeanVerificationResult( - passed=False, - toolchain=toolchain, - theorems_checked=theorem_names, - axioms_used=(), - nonstandard_axioms=(), - sorry_declarations=(), - paranoid_kernels_run=(), - sandboxed=False, - findings=tuple(findings), - caveats=(), - challenge_sorry_whitelisted_module=challenge_mod or None, - ) - caveats.append("UNSANDBOXED_EXECUTION") - - # Static source lint on every `.lean` file in the project outside `.lake` and `.git` - for lean_file in sorted(proj.rglob("*.lean")): - rel = lean_file.relative_to(proj) - if any(part in (".lake", ".git") for part in rel.parts): + caveats: tuple[str, ...] = () + for rel, data in sorted(project_files.items()): + if not rel.endswith(".lean"): continue - is_challenge = challenge_rel is not None and rel == challenge_rel - lint = lint_lean4_source( - lean_file.read_text(encoding="utf-8"), - label=rel.as_posix(), - strict_init_only=strict_init_only, - allow_challenge_sorry=is_challenge, - check_vacuous=True, - require_theorem=False, - ) - findings.extend(lint["findings"]) + findings.extend(_command_findings(rel, data.decode("utf-8"))) + if any(finding_blocks_comparator(item) for item in findings): + return _stopped() - # Scrub challenge `.olean`/`.ir` before `lake build` so `Spec` always compiles fresh from disk - scrub_challenge_artifacts(proj, challenge_mod) + challenge_key = challenge_rel.as_posix() if challenge_rel is not None else None - # Run `lake build` - build_res = subprocess.run( - [lake_bin, "build"], - cwd=proj, - env=env, - capture_output=True, - text=True, - timeout=timeout_sec, - check=False, - ) - if build_res.returncode != 0: - tail = " | ".join(line.strip() for line in ((build_res.stdout or "") + "\n" + (build_res.stderr or "")).splitlines()[-6:] if line.strip()) + def _lint_project() -> None: + for rel, data in sorted(project_files.items()): + if not rel.endswith(".lean"): + continue + lint = lint_lean4_source( + data.decode("utf-8"), + label=rel, + strict_init_only=strict_init_only, + allow_challenge_sorry=rel == challenge_key, + check_vacuous=True, + require_theorem=False, + ) + findings.extend(lint["findings"]) + + # Candidate Lean never runs on the host. The flag does not open that path. + # A missing auditor must not hide this refusal; when the auditor is present its + # findings are kept, including a planted axiom. + if not linux_bwrap_functional(): + try: + _lint_project() + except FileNotFoundError: + pass findings.append({ "line": 1, - "rule": "LAKE_BUILD_FAILED", - "message": f"`lake build` failed: {tail}", + "rule": "SANDBOX_UNAVAILABLE", + "message": ( + "Linux bubblewrap (`bwrap`) is unavailable; refusing to run candidate Lean on the host" + + (" even though unsandboxed execution was requested." if allow_unsandboxed else ".") + ), }) - return LeanVerificationResult( - passed=False, - toolchain=toolchain, - theorems_checked=theorem_names, - axioms_used=(), - nonstandard_axioms=(), - sorry_declarations=(), - paranoid_kernels_run=(), - sandboxed=bwrap_available, - findings=tuple(findings), - caveats=tuple(caveats), - challenge_sorry_whitelisted_module=challenge_mod or None, - ) + return _stopped() - # Measure actual transitive kernel axioms and sorry dependencies on solution_module via `collectAxioms` - measured_axioms: set[str] = set() - sorry_decls: list[str] = [] - nonstd: list[str] = [] - lib_olean_dir = proj / ".lake" / "build" / "lib" / "lean" - with tempfile.TemporaryDirectory(prefix="mcproof_proj_ax_") as ax_tmp_str: - ax_tmp = Path(ax_tmp_str) - audit_driver = ax_tmp / "McproofProjectAxiomDriver.lean" - audit_driver.write_text( - MODULE_AXIOM_AUDIT_TEMPLATE.format(module_name=solution_mod), - encoding="utf-8", - ) - ax_env = dict(env) - lean_path_parts = [str(ax_tmp)] - if lib_olean_dir.is_dir(): - lean_path_parts.append(str(lib_olean_dir)) - ax_env["LEAN_PATH"] = os.pathsep.join(lean_path_parts) - ax_run = subprocess.run( - [lean_bin, str(audit_driver)], - cwd=ax_tmp, - env=ax_env, - capture_output=True, - text=True, - timeout=timeout_sec, - check=False, + _lint_project() + if any(finding_blocks_comparator(item) for item in findings): + return _stopped() + + lean_bin = find_lean_tool("lean") + lake_bin = find_lean_tool("lake") + env = _lean_env() + trusted_files = _trusted_inputs(project_files) + + def refuse_sandbox(message: str) -> LeanVerificationResult: + findings.append({"line": 1, "rule": "SANDBOX_UNAVAILABLE", "message": message}) + return _stopped() + + build_dir = _materialize_trusted_tree(trusted_files) + try: + scrub_challenge_artifacts(build_dir, challenge_mod) + try: + build_res = run_confined_command( + [lake_bin, "build"], + cwd=build_dir, + env=env, + timeout_sec=timeout_sec, + ) + except CandidateSandboxUnavailable as exc: + return refuse_sandbox(str(exc)) + if build_res.returncode != 0: + tail = " | ".join( + line.strip() + for line in ((build_res.stdout or "") + "\n" + (build_res.stderr or "")).splitlines()[-6:] + if line.strip() + ) + findings.append({ + "line": 1, + "rule": "LAKE_BUILD_FAILED", + "message": f"`lake build` failed: {tail}", + }) + return _stopped(sandboxed=True) + measured_axioms, sorry_decls, nonstd, axiom_failed = _audit_solution_axioms( + build_dir, + solution_mod=solution_mod, + permitted=permitted, + lean_bin=lean_bin, + env=env, + timeout_sec=timeout_sec, + findings=findings, ) - if ax_run.returncode != 0: - tail = " | ".join(line.strip() for line in ((ax_run.stdout or "") + "\n" + (ax_run.stderr or "")).splitlines()[-4:] if line.strip()) + if axiom_failed is not None: + return refuse_sandbox(axiom_failed) + finally: + shutil.rmtree(build_dir, ignore_errors=True) + + # A second copy. The build directory now holds a manifest and anything the + # candidate planted; the comparator must not see either. + comp_dir = _materialize_trusted_tree(trusted_files) + try: + # `lake comparator` will not create a manifest inside its sandbox. Resolve one + # from this clean tree. The candidate build's manifest stays in the other directory. + try: + update_res = run_confined_command( + [lake_bin, "update"], + cwd=comp_dir, + env=env, + timeout_sec=timeout_sec, + ) + except CandidateSandboxUnavailable as exc: + return refuse_sandbox(str(exc)) + if update_res.returncode != 0: + tail = " | ".join( + line.strip() + for line in ((update_res.stdout or "") + "\n" + (update_res.stderr or "")).splitlines()[-6:] + if line.strip() + ) findings.append({ "line": 1, - "rule": "AXIOM_WALK_FAILED", - "message": f"trusted collectAxioms driver failed on {solution_mod}: {tail}", + "rule": "LAKE_BUILD_FAILED", + "message": f"`lake update` failed: {tail}", + }) + return _stopped(sandboxed=True) + try: + cmp_res = run_confined_command( + [lake_bin, "comparator", "--paranoid"], + cwd=comp_dir, + env=env, + timeout_sec=timeout_sec, + ) + except CandidateSandboxUnavailable as exc: + return refuse_sandbox(str(exc)) + kernels_accepted, solution_ok, spoof_findings = parse_comparator_stdout(cmp_res.stdout or "", solution_mod) + findings.extend(spoof_findings) + + cmp_out = (cmp_res.stdout or "") + "\n" + (cmp_res.stderr or "") + if cmp_res.returncode != 0 or not solution_ok: + if "sorryAx" in cmp_out and "sorryAx" not in sorry_decls: + sorry_decls.append("sorryAx") + for ax_m in re.finditer(r"Illegal axiom detected:\s*'([^']+)'", cmp_out): + ax_name = ax_m.group(1) + if ax_name == "sorryAx": + if ax_name not in sorry_decls: + sorry_decls.append(ax_name) + else: + nonstd.append(ax_name) + tail = " | ".join(line.strip() for line in cmp_out.strip().splitlines()[-6:] if line.strip()) + findings.append({ + "line": 1, + "rule": "COMPARATOR_PARANOID_REJECT", + "message": f"`lake comparator --paranoid` rejected project (exit {cmp_res.returncode}): {tail}", }) - else: - for line in (ax_run.stdout or "").splitlines(): - if not line.startswith("MCPROOF_AUDIT\t"): - continue - parts = line.split("\t") - if len(parts) != 5: - continue - _, kind, _, decl_name, ax_csv = parts - axs = {a for a in ax_csv.split(",") if a} - measured_axioms.update(axs) - if kind in ("axiom", "opaque"): - findings.append({ - "line": 1, - "rule": "KERNEL_EXTRA_AXIOM_OR_OPAQUE", - "message": f"solution module {solution_mod} contains {kind} declaration '{decl_name}'", - }) - if "sorryAx" in axs: - sorry_decls.append(decl_name) - findings.append({ - "line": 1, - "rule": "KERNEL_SORRY_AXIOM", - "message": f"solution declaration '{decl_name}' depends on 'sorryAx'", - }) - extra_axs = axs - (permitted & STANDARD_AXIOMS) - {"sorryAx"} - if extra_axs: - nonstd.extend(sorted(extra_axs)) - findings.append({ - "line": 1, - "rule": "KERNEL_NONSTANDARD_AXIOM", - "message": f"solution declaration '{decl_name}' uses unpermitted axioms: {sorted(extra_axs)}", - }) - - cmp_cmd = [lake_bin, "comparator", "--paranoid"] - if not bwrap_available: - cmp_cmd.append("--inadvisably-no-sandbox") - - cmp_res = subprocess.run( - cmp_cmd, - cwd=proj, - env=env, - capture_output=True, - text=True, - timeout=timeout_sec, - check=False, - ) - kernels_accepted, solution_ok, spoof_findings = parse_comparator_stdout(cmp_res.stdout or "", solution_mod) - findings.extend(spoof_findings) - - cmp_out = (cmp_res.stdout or "") + "\n" + (cmp_res.stderr or "") - if cmp_res.returncode != 0 or not solution_ok: - if "sorryAx" in cmp_out and "sorryAx" not in sorry_decls: - sorry_decls.append("sorryAx") - for ax_m in re.finditer(r"Illegal axiom detected:\s*'([^']+)'", cmp_out): - ax_name = ax_m.group(1) - if ax_name == "sorryAx": - if ax_name not in sorry_decls: - sorry_decls.append(ax_name) - else: - nonstd.append(ax_name) - tail = " | ".join(line.strip() for line in cmp_out.strip().splitlines()[-6:] if line.strip()) - findings.append({ - "line": 1, - "rule": "COMPARATOR_PARANOID_REJECT", - "message": f"`lake comparator --paranoid` rejected project (exit {cmp_res.returncode}): {tail}", - }) - passed = ( - len(findings) == 0 - and cmp_res.returncode == 0 - and solution_ok - and len(kernels_accepted) == 6 - ) - return LeanVerificationResult( - passed=passed, - toolchain=toolchain, - theorems_checked=theorem_names, - axioms_used=tuple(sorted(measured_axioms - {"sorryAx"})), - nonstandard_axioms=tuple(sorted(set(nonstd))), - sorry_declarations=tuple(sorted(set(sorry_decls))), - paranoid_kernels_run=tuple(kernels_accepted), - sandboxed=bwrap_available, - findings=tuple(findings), - caveats=tuple(caveats), - challenge_sorry_whitelisted_module=challenge_mod or None, - ) + passed = ( + len(findings) == 0 + and cmp_res.returncode == 0 + and solution_ok + and len(kernels_accepted) == 6 + ) + return LeanVerificationResult( + passed=passed, + toolchain=toolchain, + theorems_checked=theorem_names, + axioms_used=tuple(sorted(measured_axioms - {"sorryAx"})), + nonstandard_axioms=tuple(sorted(set(nonstd))), + sorry_declarations=tuple(sorted(set(sorry_decls))), + paranoid_kernels_run=tuple(kernels_accepted), + sandboxed=True, + findings=tuple(findings), + caveats=caveats, + challenge_sorry_whitelisted_module=challenge_mod or None, + ) + finally: + shutil.rmtree(comp_dir, ignore_errors=True) diff --git a/src/mcproof/repair_loop.py b/src/mcproof/repair_loop.py index d2c52e9..53589e2 100644 --- a/src/mcproof/repair_loop.py +++ b/src/mcproof/repair_loop.py @@ -33,9 +33,21 @@ import urllib.request from .lean_harness import ( + BAR_FORBIDDEN_RULES, THEOREM_DECL_RE, + CandidateSandboxUnavailable, + UntrustedProjectPath, + _lake_symlink_finding, + _lean_commands_that_may_run, + _lean_env, + _materialize_trusted_tree, + _read_project_tree, + _trusted_inputs, + linux_bwrap_functional, + run_confined_command, strip_lean_comments_and_strings, verify_lean4_against_challenge, + verify_lean4_project_comparator, verify_lean4_prompt_obligations, verify_pinned_lean_defs, ) @@ -52,7 +64,7 @@ run_verdicts_oracle, sanitize_verdict_id, ) -from .paths import dep_repo, repo_root +from .paths import dep_repo, find_lean_tool, repo_root MAX_RETRIEVAL_TOKENS_HARD_CAP = 1500 QWEN35_STOP_TOKENS: tuple[str, ...] = ("<|im_end|>", "<|endoftext|>") @@ -819,77 +831,531 @@ def _extract_first_json_block(text: str) -> str | None: return blocks[0].strip() if blocks else None -def _run_public_family_c_harness(harness: Mapping[str, Any], completion_text: str) -> list[dict[str, Any]]: - """Post-hoc Family C ground-truth scorer matching `datasets/scripts/grade_item.py`.""" - otype = harness.get("oracle_type") - meta = harness.get("spec_meta") or {} +_FAMILY_C_STATUS_VERIFIED_RE = re.compile(r"(?im)^status\s*:\s*verified\b") +_FAMILY_C_STATUS_NOT_VERIFIED_RE = re.compile(r"(?im)^status\s*:.*not verified") +_FAMILY_C_SORRY_OR_ADMIT_RE = re.compile(r"\b(sorry|admit)\b") +# Lean atomic names include `«quoted»` forms and non-ASCII letters such as `α` and `café`. +_LEAN_NAME_ATOM = r"(?:«[^»]*»|_*(?:[^\W\d_])[\w!?']*)" +_LEAN_NAME = rf"{_LEAN_NAME_ATOM}(?:\.{_LEAN_NAME_ATOM})*" +_FAMILY_C_DECL_NAME_RE = re.compile( + rf"\b(?:theorem|lemma)\b[ \t]+(?P{_LEAN_NAME})" +) +_FAMILY_C_OTHER_DECL_RE = re.compile( + rf"(?{_LEAN_NAME}))?(?=[ \t]*:)" + rf"|example\b(?:[ \t]+(?P{_LEAN_NAME}))?(?=[ \t]*:)" + rf"|abbrev\b[ \t]+(?P{_LEAN_NAME})" + rf")" +) + + +def _normalize_family_c_statement(text: str) -> str: + """Collapse whitespace so statement hashes are layout-insensitive.""" + return re.sub(r"\s+", " ", text).strip() + + +def _family_c_candidate_statements(candidate_lean: str) -> dict[str, str]: + """Map each fully-qualified theorem name to the sha256 of its normalized statement.""" + cleaned = strip_lean_comments_and_strings(candidate_lean) + stmts: dict[str, str] = {} + for match in THEOREM_DECL_RE.finditer(cleaned): + full = str(match.group("name")) + stmt = _normalize_family_c_statement(f"{full} {match.group('header')}") + stmts[full] = hashlib.sha256(stmt.encode("utf-8")).hexdigest() + return stmts + + +def _family_c_all_decl_names(candidate_lean: str) -> set[str]: + """Declaration names the line-start ASCII theorem regex does not keep. + + Includes a mid-line `theorem`, a `«quoted»` or non-ASCII name, and + `instance`, `example`, and `abbrev`. An unnamed `example` or `instance` + is recorded under that keyword. + """ + cleaned = strip_lean_comments_and_strings(candidate_lean) + names = {str(match.group("name")) for match in _FAMILY_C_DECL_NAME_RE.finditer(cleaned)} + for match in _FAMILY_C_OTHER_DECL_RE.finditer(cleaned): + if match.group(0).startswith("instance"): + names.add(str(match.group("inst") or "instance")) + elif match.group(0).startswith("example"): + names.add(str(match.group("ex") or "example")) + elif match.group("abbrev"): + names.add(str(match.group("abbrev"))) + return names + + +def _family_c_forbidden_eval_findings(candidate_lean: str) -> list[dict[str, Any]]: + """Reject every command in the `FORBIDDEN_EVAL_COMMAND` lint, including mid-line. + + The scan uses that lint's own pattern on comment-stripped and string-stripped + source. A hit returns before `lake build`, because those commands run candidate + code during elaboration. + """ + cleaned = strip_lean_comments_and_strings(candidate_lean) + rx = dict(BAR_FORBIDDEN_RULES)["FORBIDDEN_EVAL_COMMAND"] + found: list[str] = [] + for match in rx.finditer(cleaned): + cmd = match.group(0) + if cmd not in found: + found.append(cmd) + if not found: + return [] + return [ + { + "rule": "FORBIDDEN_EVAL_COMMAND", + "message": ( + "candidate contains a Lean command that runs during elaboration: " + + ", ".join(found) + + "." + ), + } + ] + + +# `initialize` / `builtin_initialize` and the `@[init]` / `@[builtin_init]` attributes run IO +# during `lake build`. `run_cmd`, `run_elab`, `run_tac`, and `elab` do too, before the +# comparator's own lint. Identifier characters include `'`, `.`, and `«»`, which are not +# `\w`, so the boundary is explicit: `elab'`, `x.elab`, and `«elab»` stay legal names. +_FAMILY_C_NAME_EDGE = r"\w.'»«" +_FAMILY_C_INITIALIZER_RE = re.compile( + rf"(? list[str]: + found: list[str] = [] + for match in rx.finditer(cleaned): + cmd = match.group(0) + if cmd not in found: + found.append(cmd) + return found + + +def _family_c_initializer_findings(candidate_lean: str) -> list[dict[str, Any]]: + """Reject initializers before `lake build`. They execute IO while the project builds.""" + cleaned = strip_lean_comments_and_strings(candidate_lean) + found = _family_c_unique_matches(_FAMILY_C_INITIALIZER_RE, cleaned) + if not found: + return [] + return [ + { + "rule": "FORBIDDEN_INITIALIZER", + "message": ( + "candidate contains a Lean initializer that runs during lake build: " + + ", ".join(found) + + "." + ), + } + ] + + +def _family_c_elab_command_findings(candidate_lean: str) -> list[dict[str, Any]]: + """Reject elaboration commands before the first `lake build`, not only in the comparator.""" + cleaned = strip_lean_comments_and_strings(candidate_lean) + found = _family_c_unique_matches(_FAMILY_C_ELAB_COMMAND_RE, cleaned) + if not found: + return [] + return [ + { + "rule": "FORBIDDEN_ELAB_COMMAND", + "message": ( + "candidate contains a Lean elaboration command that runs during lake build: " + + ", ".join(found) + + "." + ), + } + ] + + +def _family_c_build_side_effect_findings(candidate_lean: str) -> list[dict[str, Any]]: + """Record `attribute [init]`, `run_meta`, and `elab_rules` without skipping the build. + + The sandbox is what stops the write. These findings stay on the result either way. + """ + cleaned = strip_lean_comments_and_strings(candidate_lean) + found = _family_c_unique_matches(_FAMILY_C_BUILD_SIDE_EFFECT_RE, cleaned) + if not found: + return [] + return [ + { + "rule": "FORBIDDEN_BUILD_SIDE_EFFECT", + "message": ( + "candidate contains a command that runs during compilation or import: " + + ", ".join(found) + + "." + ), + } + ] + + +def _family_c_statement_headers( + meta: Mapping[str, Any], theorem_names: Sequence[str] +) -> dict[str, str] | None: + """Return the pinned header of every challenge theorem, or None if any pin is missing. + + A header is the text after the fully-qualified theorem name, the same span + `THEOREM_DECL_RE` captures. Callers hash that text. Nothing here invents `op x = x`. + """ + raw = meta.get("statements") + if not isinstance(raw, Mapping): + return None + headers: dict[str, str] = {} + for name in theorem_names: + full = str(name) + header = raw.get(full) + if not isinstance(header, str) or not header.strip(): + return None + headers[full] = header.strip() + return headers + + +def _family_c_expected_statements(headers: Mapping[str, str]) -> dict[str, str]: + """Map each fully-qualified name to the sha256 of its pinned, whitespace-normalized statement.""" + expected: dict[str, str] = {} + for full, header in headers.items(): + stmt = _normalize_family_c_statement(f"{full} {header}") + expected[str(full)] = hashlib.sha256(stmt.encode("utf-8")).hexdigest() + return expected + + +def _family_c_root_source(pkg: str, meta: Mapping[str, Any]) -> str: + """Library module for the generated project. Uses caller-supplied defs, never an identity `op`.""" + defs = meta.get("definitions") + if isinstance(defs, str) and defs.strip(): + return defs.strip() + "\n" + return f"namespace {pkg}\nend {pkg}\n" + + +def _family_c_spec_source(pkg: str, headers: Mapping[str, str]) -> str: + """Challenge module: each pinned header, proved with `sorry` so the solution file is what `lake` checks.""" + lines = [f"import {pkg}", f"namespace {pkg}.Spec"] + for full, header in headers.items(): + short = str(full).split(".")[-1] + lines.append(f"theorem {short} {header} := by sorry") + lines.append(f"end {pkg}.Spec") + return "\n".join(lines) + "\n" + + +def _family_c_lakefile(pkg: str) -> str: + """Build the library, the challenge, and the candidate proofs. `["{pkg}"]` alone skips the proofs.""" + return ( + f'name = "{pkg.lower()}"\nversion = "0.1.0"\n' + f'defaultTargets = ["{pkg}", "{pkg}.Spec", "{pkg}.Proofs"]\n\n' + f'[[lean_lib]]\nname = "{pkg}"\n\n[[lean_lib]]\nname = "{pkg}.Spec"\n\n' + f'[[lean_lib]]\nname = "{pkg}.Proofs"\n\n[[lean_exe]]\nname = "{pkg.lower()}"\nroot = "Main"\n' + ) + + +def _family_c_code_under_proof(pkg: str, meta: Mapping[str, Any]) -> list[str]: + raw = meta.get("code_under_proof") + if isinstance(raw, list) and raw and all(isinstance(item, str) and item.strip() for item in raw): + return [str(item) for item in raw] + return [pkg] + + +def _family_c_toolchain_pin() -> str: + return (repo_root() / "lean" / "lean-toolchain").read_text(encoding="utf-8").strip() + "\n" + + +def _check_family_c_statement_match( + meta: Mapping[str, Any], + theorem_names: Sequence[str], + candidate_lean: str, +) -> list[dict[str, Any]]: + """Statement-match guard: names plus hashes of the pinned target statements.""" findings: list[dict[str, Any]] = [] + headers = _family_c_statement_headers(meta, theorem_names) + if headers is None: + findings.append( + { + "rule": "FAMILY_C_TARGET_STATEMENT_ABSENT", + "message": ( + "spec_meta.statements must pin the header of every theorem_names entry. " + "The turn does not invent an identity statement." + ), + } + ) + return findings + cand = _family_c_candidate_statements(candidate_lean) + exp = _family_c_expected_statements(headers) + hidden = _family_c_all_decl_names(candidate_lean) - set(cand) + if hidden or set(cand) != set(exp): + findings.append( + { + "rule": "FAMILY_C_THEOREM_NAME_MISMATCH", + "message": ( + "candidate theorem names do not equal challenge meta: " + f"missing={sorted(set(exp) - set(cand))} " + f"extra={sorted((set(cand) - set(exp)) | hidden)}." + ), + } + ) + return findings + for name in sorted(exp): + if cand[name] != exp[name]: + findings.append( + { + "rule": "FAMILY_C_STATEMENT_MISMATCH", + "message": ( + f"candidate statement hash for '{name}' does not equal challenge meta; " + "statements must match exactly." + ), + } + ) + return findings - if otype == "bar_check": - bar_tpl = dep_repo("mission") / "bar" / "template" - bar_chk = dep_repo("mission") / "bar" / "check.py" - pkg = str(meta["pkg"]) - thms = list(meta["theorem_names"]) - readme_body = re.sub(r"(?im)^Claim-Tier:[^\n]*\n+", "", completion_text, count=1) - says_verified = bool( - re.search(r"(?im)^status\s*:\s*verified\b", readme_body) - and not re.search(r"(?im)^status\s*:.*not verified", readme_body) + +def _family_c_lake_failure_message(stdout: str, stderr: str) -> str: + """Keep the compiler `file:line:column` lines, then the short lake tail.""" + lines = [line.strip() for line in f"{stdout}\n{stderr}".splitlines() if line.strip()] + located = [line for line in lines if re.search(r"\.lean:\d+:\d+", line)] + tail = lines[-6:] + kept: list[str] = [] + for line in located + tail: + if line not in kept: + kept.append(line) + return " | ".join(kept[-8:]) + + +def _family_c_sandbox_unavailable_finding() -> dict[str, Any]: + return { + "rule": "SANDBOX_UNAVAILABLE", + "message": ( + "Linux bubblewrap (`bwrap`) is unavailable; " + "refusing to compile or import candidate Lean unsandboxed." + ), + } + + +def _family_c_direct_lake_build(root: Path) -> list[dict[str, Any]]: + """Run `lake build` on candidate text inside the bubblewrap sandbox. + + A missing `soko` tree must not hide a Lean error. A missing `lean`/`lake` still fails closed. + A missing sandbox refuses the build. There is no unsandboxed fallback. + """ + lake_hit = _lake_symlink_finding(root) + if lake_hit is not None: + return [lake_hit] + if not linux_bwrap_functional(): + return [_family_c_sandbox_unavailable_finding()] + try: + lake_bin = find_lean_tool("lake") + env = _lean_env() + except (OSError, RuntimeError) as exc: + return [ + { + "rule": "FAMILY_C_LEAN_TOOLCHAIN_UNAVAILABLE", + "message": f"Lean toolchain unavailable; turn fails closed: {exc}", + } + ] + try: + build_res = run_confined_command( + [lake_bin, "build"], + cwd=root, + env=env, + timeout_sec=300, + ) + except CandidateSandboxUnavailable: + return [_family_c_sandbox_unavailable_finding()] + if build_res.returncode != 0: + return [ + { + "rule": "LAKE_BUILD_FAILED", + "message": f"`lake build` failed: {_family_c_lake_failure_message(build_res.stdout or '', build_res.stderr or '')}", + } + ] + return [] + + +def _lean_exception_is_missing_toolchain(exc: BaseException) -> bool: + message = str(exc) + return "Lean tool" in message or "elan bin" in message + + +def _run_family_c_real_lean_turn(harness: Mapping[str, Any], completion_text: str) -> list[dict[str, Any]]: + """Real-Lean Family C turn for `oracle_type: bar_check` only. + + `bar_status` comes from harness meta. The challenge statements come from + `spec_meta.statements`. The solution file is the candidate's fenced lean. + `verdict_audit` items never enter this function. `lake build` still runs when + the mission template or the soko auditor checkout is absent. It does not run + when `lean` and `lake` themselves are absent. + """ + meta = harness.get("spec_meta") or {} + pkg = str(meta.get("pkg", "")).strip() + thms = [str(t) for t in (meta.get("theorem_names") or [])] + # Bar status comes from harness meta exclusively; candidate prose can never promote it. + bar_status = str(meta.get("bar_status", "")).strip() + if not pkg or not thms: + return [ + { + "rule": "FAMILY_C_MALFORMED_HARNESS", + "message": "harness spec_meta must define pkg and theorem_names.", + } + ] + findings: list[dict[str, Any]] = [] + readme_body = re.sub(r"(?im)^Claim-Tier:[^\n]*\n+", "", completion_text, count=1) + if ( + _FAMILY_C_STATUS_VERIFIED_RE.search(readme_body) + and not _FAMILY_C_STATUS_NOT_VERIFIED_RE.search(readme_body) + and bar_status.lower() != "verified" + ): + findings.append( + { + "rule": "FAMILY_C_STATUS_SELF_ATTESTATION", + "message": ( + f"candidate declares 'Status: verified' but harness meta bar_status is {bar_status!r}; " + "bar status comes from harness meta exclusively." + ), + } ) - bar_status = "verified" if says_verified else str(meta["bar_status"]) - with tempfile.TemporaryDirectory(prefix="mcproof_rep_bar_") as td: - root = Path(td) - shutil.copytree(bar_tpl, root, dirs_exist_ok=True) + lean_blocks = extract_fenced_blocks(completion_text, ("lean", "lean4")) + candidate_lean = "\n\n".join(block.strip() for block in lean_blocks if block.strip()) + if not candidate_lean: + findings.append( + { + "rule": "FAMILY_C_MISSING_CANDIDATE_LEAN", + "message": "no ```lean code blocks found; the checked project is built from the candidate's lean blocks.", + } + ) + return findings + if _FAMILY_C_SORRY_OR_ADMIT_RE.search(strip_lean_comments_and_strings(candidate_lean)): + findings.append( + { + "rule": "FAMILY_C_SOLUTION_USES_SORRY_OR_ADMIT", + "message": "solution module uses 'sorry' or 'admit'; the turn fails.", + } + ) + findings.extend(_family_c_forbidden_eval_findings(candidate_lean)) + findings.extend(_family_c_initializer_findings(candidate_lean)) + findings.extend(_family_c_elab_command_findings(candidate_lean)) + for cmd in _lean_commands_that_may_run(candidate_lean): + if any(cmd in str(item.get("message", "")) for item in findings): + continue + if cmd.startswith("#"): + findings.append({ + "rule": "FORBIDDEN_EVAL_COMMAND", + "message": "candidate contains a Lean command that runs during elaboration: " + cmd + ".", + }) + else: + findings.append({ + "rule": "FORBIDDEN_ELAB_COMMAND", + "message": "candidate contains a Lean elaboration command that runs during lake build: " + cmd + ".", + }) + findings.extend(_check_family_c_statement_match(meta, thms, candidate_lean)) + if findings: + return findings + headers = _family_c_statement_headers(meta, thms) + if headers is None: + return [ + { + "rule": "FAMILY_C_TARGET_STATEMENT_ABSENT", + "message": ( + "spec_meta.statements must pin the header of every theorem_names entry. " + "The turn does not invent an identity statement." + ), + } + ] + mission_bar: tuple[Path, Path] | None = None + try: + mission = dep_repo("mission") + except FileNotFoundError as exc: + findings.append( + { + "rule": "FAMILY_C_MISSION_BAR_UNAVAILABLE", + "message": ( + "mission bar template is unavailable, so the structural bar replay was not run. " + f"{exc}" + ), + } + ) + else: + bar_tpl = mission / "bar" / "template" + bar_chk = mission / "bar" / "check.py" + if bar_tpl.is_dir() and bar_chk.is_file(): + mission_bar = (bar_tpl, bar_chk) + else: + findings.append( + { + "rule": "FAMILY_C_MISSION_BAR_UNAVAILABLE", + "message": f"mission checkout at {mission} has no bar/template or bar/check.py.", + } + ) + code_toml = ", ".join(json.dumps(name) for name in _family_c_code_under_proof(pkg, meta)) + test_cmd = str(meta.get("test_cmd") or "python3 tests/run_tests.py") + with tempfile.TemporaryDirectory(prefix="mcproof_rep_bar_") as td: + root = Path(td) + if mission_bar is not None: + shutil.copytree(mission_bar[0], root, dirs_exist_ok=True) (root / "Clamp.lean").unlink(missing_ok=True) shutil.rmtree(root / "Clamp", ignore_errors=True) - (root / f"{pkg}.lean").write_text( - f"namespace {pkg}\ndef op (x : Nat) : Nat := x\nend {pkg}\n", - encoding="utf-8", - ) - (root / pkg).mkdir(exist_ok=True) - spec_lines = [f"import {pkg}", f"namespace {pkg}.Spec"] - proof_lines = [f"import {pkg}", f"namespace {pkg}.Spec"] - for tfull in thms: - tshort = str(tfull).split(".")[-1] - spec_lines.append(f"theorem {tshort} (x : Nat) : {pkg}.op x = x := by sorry") - proof_lines.append(f"theorem {tshort} (x : Nat) : {pkg}.op x = x := rfl") - spec_lines.append(f"end {pkg}.Spec\n") - proof_lines.append(f"end {pkg}.Spec\n") - (root / pkg / "Spec.lean").write_text("\n".join(spec_lines), encoding="utf-8") - (root / pkg / "Proofs.lean").write_text("\n".join(proof_lines), encoding="utf-8") - (root / "Main.lean").write_text( - f'import {pkg}\ndef main : IO Unit := IO.println s!"{{{pkg}.op 1}}"\n', - encoding="utf-8", - ) - (root / "lakefile.toml").write_text( - f'name = "{pkg.lower()}"\nversion = "0.1.0"\ndefaultTargets = ["{pkg}"]\n\n' - f'[[lean_lib]]\nname = "{pkg}"\n\n[[lean_lib]]\nname = "{pkg}.Spec"\n\n' - f'[[lean_lib]]\nname = "{pkg}.Proofs"\n\n[[lean_exe]]\nname = "{pkg.lower()}"\nroot = "Main"\n', - encoding="utf-8", - ) - (root / "comparator.json").write_text( - json.dumps( - { - "challenge_module": f"{pkg}.Spec", - "solution_module": f"{pkg}.Proofs", - "theorem_names": thms, - "permitted_axioms": ["propext", "Quot.sound", "Classical.choice"], - }, - indent=2, - ) - + "\n", - encoding="utf-8", + # Keep lake-manifest.json. bar/check.py item 3 fails closed when it is + # absent, including under --no-lean, so deleting it rejects a correct proof. + else: + (root / "lean-toolchain").write_text(_family_c_toolchain_pin(), encoding="utf-8") + (root / f"{pkg}.lean").write_text(_family_c_root_source(pkg, meta), encoding="utf-8") + (root / pkg).mkdir(exist_ok=True) + (root / pkg / "Spec.lean").write_text(_family_c_spec_source(pkg, headers), encoding="utf-8") + # Solution module is the candidate's lean blocks verbatim. Theorems must use + # the fully-qualified challenge name (for example `theorem CrtPair.Spec.crt_mod_left ...`). + proofs_path = root / pkg / "Proofs.lean" + proofs_text = f"import {pkg}\n\n{candidate_lean}\n" + proofs_path.write_text(proofs_text, encoding="utf-8") + (root / "Main.lean").write_text( + f'import {pkg}\ndef main : IO Unit := IO.println "ok"\n', + encoding="utf-8", + ) + (root / "lakefile.toml").write_text(_family_c_lakefile(pkg), encoding="utf-8") + comparator_path = root / "comparator.json" + comparator_text = ( + json.dumps( + { + "challenge_module": f"{pkg}.Spec", + "solution_module": f"{pkg}.Proofs", + "theorem_names": thms, + "permitted_axioms": ["propext", "Quot.sound", "Classical.choice"], + }, + indent=2, ) + + "\n" + ) + comparator_path.write_text(comparator_text, encoding="utf-8") + if mission_bar is not None: (root / "bar.toml").write_text( f'[project]\nname = "{pkg.lower()}"\nstatus = "{bar_status}"\npage = "README.md"\n' f'spec_authors = ["example-author"]\nentry_points = ["Main.lean"]\n\n' - f'[spec]\nfiles = ["SPEC.md", "{pkg}/Spec.lean"]\ncode = ["{pkg}.op"]\n\n' - f'[tests]\nresults = "tests/TEST-RESULTS.md"\ncommand = "{meta["test_cmd"]}"\nfiles = ["tests/run_tests.py"]\n', + f'[spec]\nfiles = ["SPEC.md", "{pkg}/Spec.lean"]\ncode = [{code_toml}]\n\n' + f'[tests]\nresults = "tests/TEST-RESULTS.md"\ncommand = "{test_cmd}"\nfiles = ["tests/run_tests.py"]\n', encoding="utf-8", ) (root / "README.md").write_text(readme_body, encoding="utf-8") out_json = root / "bar_out.json" proc = subprocess.run( - [sys.executable, "-O", "-B", "-W", "error", str(bar_chk), str(root), "--no-lean", "--offline", "--json", str(out_json)], + [ + sys.executable, + "-O", + "-B", + "-W", + "error", + str(mission_bar[1]), + str(root), + "--no-lean", + "--offline", + "--json", + str(out_json), + ], capture_output=True, text=True, check=False, @@ -902,7 +1368,79 @@ def _run_public_family_c_harness(harness: Mapping[str, Any], completion_text: st for req_i in (2, 4, 5, 6): if by_item.get(req_i) != "PASS": findings.append({"rule": "BAR_CHECK_ITEM_FAIL", "message": f"bar item {req_i} did not pass"}) - return findings + # `lake build` is the formal signal. Its message keeps `.lean:line:column` lines. + # The comparator's own tail keeps only the last 6 lines, which can drop that error. + # Kernel checks run only after this build succeeds, and they run on a fresh + # copy of the files taken before the build. The candidate tree is not written again. + lake_hit = _lake_symlink_finding(root) + if lake_hit is not None: + findings.append(lake_hit) + return findings + try: + trusted_inputs = _trusted_inputs(_read_project_tree(root)) + except UntrustedProjectPath as exc: + findings.append({"rule": exc.rule, "message": str(exc)}) + return findings + findings.extend(_family_c_build_side_effect_findings(candidate_lean)) + findings.extend(_family_c_direct_lake_build(root)) + if any( + f.get("rule") + in { + "LAKE_BUILD_FAILED", + "FAMILY_C_LEAN_TOOLCHAIN_UNAVAILABLE", + "SANDBOX_UNAVAILABLE", + "LAKE_CACHE_SYMLINK", + "UNTRUSTED_PROJECT_SYMLINK", + "UNTRUSTED_PROJECT_FILE", + "NOFOLLOW_OPEN_UNAVAILABLE", + } + for f in findings + ): + return findings + fresh = _materialize_trusted_tree(trusted_inputs) + try: + try: + lean_res = verify_lean4_project_comparator(fresh, allow_unsandboxed=False) + except FileNotFoundError as exc: + if _lean_exception_is_missing_toolchain(exc): + findings.append( + { + "rule": "FAMILY_C_LEAN_TOOLCHAIN_UNAVAILABLE", + "message": f"Lean toolchain unavailable; turn fails closed: {exc}", + } + ) + return findings + findings.append( + { + "rule": "FAMILY_C_LEAN_AUDIT_DEPS_UNAVAILABLE", + "message": f"lake build passed, but the comparator auditor checkout is missing: {exc}", + } + ) + return findings + except (OSError, RuntimeError) as exc: + findings.append( + { + "rule": "FAMILY_C_LEAN_TOOLCHAIN_UNAVAILABLE", + "message": f"Lean toolchain unavailable; turn fails closed: {exc}", + } + ) + return findings + for finding in lean_res.findings: + findings.append(dict(finding)) + finally: + shutil.rmtree(fresh, ignore_errors=True) + return findings + + +def _run_public_family_c_harness(harness: Mapping[str, Any], completion_text: str) -> list[dict[str, Any]]: + """Post-hoc Family C ground-truth scorer matching `datasets/scripts/grade_item.py`.""" + otype = harness.get("oracle_type") + + if otype == "bar_check": + return _run_family_c_real_lean_turn(harness, completion_text) + + meta = harness.get("spec_meta") or {} + findings: list[dict[str, Any]] = [] if otype == "verdict_audit": jb = _extract_first_json_block(completion_text) diff --git a/tests/test_candidate_confinement.py b/tests/test_candidate_confinement.py new file mode 100644 index 0000000..5031d65 --- /dev/null +++ b/tests/test_candidate_confinement.py @@ -0,0 +1,945 @@ +"""Host writes, env leaks, and trusted-input swaps must fail closed. + +Each test is aimed at one fix. Reverting that fix fails its test: +a passthrough `run_confined_command` writes the host marker, dropping `env -i` +leaks the sentinel, `--ro-bind / /` makes `.git/config` and the sockets visible, +a line-anchored `attribute` regex misses the split form, skipping the restore +lets the kernel see the rewritten Proofs file, and an uncaught `json.loads` +raises on malformed `comparator.json`. +""" + +from __future__ import annotations + +import io +import json +from pathlib import Path +import shutil +import socket +import threading +import uuid + +import pytest + +from helpers import expect_false, expect_in, expect_true, has_dep, has_lean_toolchain +from mcproof.cli import main as cli_main +from mcproof.lean_harness import ( + _NOFOLLOW_OPEN_RULE, + _SANDBOX_CPU_SEC, + UntrustedProjectPath, + _address_space_cap_bytes, + _nofollow_dir_ops_available, + _spawn_confined, + linux_bwrap_functional, + lint_lean4_source, + verify_lean4_project_comparator, + verify_lean4_source, +) +from mcproof.oracles import grade_family_completion, run_prose_and_mixed_zone_oracle +from mcproof.repair_loop import _family_c_build_side_effect_findings, _run_family_c_real_lean_turn +from test_domain_and_repair_loop import _family_c_refl_harness, _family_c_with_theorems + + +def _skip_without_confinement() -> None: + if not has_lean_toolchain(): + pytest.skip("Lean is absent, so this confinement path was not exercised") + if not linux_bwrap_functional(): + pytest.skip("bwrap confinement probe failed, so this path was not exercised") + + +def _skip_without_soko() -> None: + if not has_dep("soko"): + pytest.skip("soko is absent, so the Lean auditor path was not exercised") + + +def _hidden_eval(marker: Path) -> str: + """`#eval` hidden by a short raw-string closer. The lint does not see it, so the compile runs.""" + return ( + "theorem host_marker_witness (n : Nat) : n = n := rfl\n" + 'def s := r##"foo"#\n' + f'#eval IO.FS.writeFile "{marker}" "p"\n' + ) + + +def _expect_host_marker_absent(run) -> None: + marker = Path("/tmp") / f"mcproof-host-marker-{uuid.uuid4().hex}" + marker.unlink(missing_ok=True) + try: + run(marker) + expect_false(marker.is_file(), f"candidate wrote {marker}; the compile was not confined") + finally: + marker.unlink(missing_ok=True) + + +def test_verify_lean4_source_cannot_write_a_host_marker() -> None: + """Family A's compiler. A passthrough sandbox writes the marker and this fails.""" + _skip_without_confinement() + _skip_without_soko() + + def run(marker: Path) -> None: + verify_lean4_source(_hidden_eval(marker)) + + _expect_host_marker_absent(run) + + +def test_family_a_grader_cannot_write_a_host_marker() -> None: + _skip_without_confinement() + _skip_without_soko() + + def run(marker: Path) -> None: + grade_family_completion("A", "```lean\n" + _hidden_eval(marker) + "```\n") + + _expect_host_marker_absent(run) + + +def test_family_c_grader_cannot_write_a_host_marker() -> None: + """`grade_family_completion` for Family C runs before the sandboxed harness.""" + _skip_without_confinement() + _skip_without_soko() + + def run(marker: Path) -> None: + grade_family_completion("C", "```lean\n" + _hidden_eval(marker) + "```\n") + + _expect_host_marker_absent(run) + + +def test_family_d_embedded_lean_cannot_write_a_host_marker() -> None: + _skip_without_confinement() + _skip_without_soko() + + def run(marker: Path) -> None: + text = "```lean\n" + _hidden_eval(marker) + "```\n" + run_prose_and_mixed_zone_oracle( + text, + require_formal_sections=False, + require_data_eido_skip=False, + verify_embedded_lean=True, + verify_verdict_shas=False, + ) + grade_family_completion("D", text) + + _expect_host_marker_absent(run) + + +def test_cli_lean_cannot_write_a_host_marker(monkeypatch: pytest.MonkeyPatch) -> None: + _skip_without_confinement() + _skip_without_soko() + + def run(marker: Path) -> None: + monkeypatch.setattr("sys.stdin", io.StringIO(_hidden_eval(marker))) + cli_main(["lean", "-"]) + + _expect_host_marker_absent(run) + + +def test_run_lean4_verification_cannot_write_a_host_marker() -> None: + _skip_without_confinement() + if not has_dep("soko"): + pytest.skip("soko is absent, so run_lean4_verification was not exercised") + from verification.graders.common import run_lean4_verification + + def run(marker: Path) -> None: + run_lean4_verification(_hidden_eval(marker)) + + _expect_host_marker_absent(run) + + +def test_execution_lint_stops_verify_lean4_source_before_compile(monkeypatch: pytest.MonkeyPatch) -> None: + """`attribute [init]` is a lint finding. Removing the early return calls Lean and this fails.""" + _skip_without_soko() + + def boom(*_args: object, **_kwargs: object) -> None: + raise AssertionError("Lean ran after an execution lint") + + monkeypatch.setattr("mcproof.lean_harness.run_confined_command", boom) + monkeypatch.setattr("mcproof.lean_harness.subprocess.run", boom) + res = verify_lean4_source( + 'def pwnInit : IO Unit := pure ()\nattribute [init] pwnInit\n', + require_theorem=False, + ) + expect_in("FORBIDDEN_INIT_ATTRIBUTE", {f["rule"] for f in res.findings}) + + +def test_execution_lint_stops_run_lean4_verification_before_compile( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """soko accepts `attribute [init]`. Removing this lint stop compiles it and this fails.""" + if not has_dep("soko"): + pytest.skip("soko is absent, so run_lean4_verification was not exercised") + from verification.graders.common import run_lean4_verification + + def boom(*_args: object, **_kwargs: object) -> None: + raise AssertionError("Lean ran after an execution lint") + + monkeypatch.setattr("mcproof.lean_harness.run_confined_command", boom) + monkeypatch.setattr("verification.graders.common.run_confined_command", boom) + rep = run_lean4_verification('def pwnInit : IO Unit := pure ()\nattribute [init] pwnInit\n') + expect_false(rep["ok"], str(rep)) + expect_true(any(str(err).startswith("lint:FORBIDDEN_INIT_ATTRIBUTE") for err in rep["errors"]), rep) + + +def _skip_without_bwrap() -> None: + if not shutil.which("bwrap"): + pytest.skip("bwrap is absent, so this confinement check was not exercised") + + +def test_confined_process_does_not_leak_parent_environ(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Dropping `env -i` puts this sentinel in `/proc/1/environ` and this fails.""" + _skip_without_bwrap() + monkeypatch.setenv("MCPROOF_SANDBOX_SENTINEL", "leakme-parent-env") + monkeypatch.setenv("GIT_CONFIG_VALUE_0", "supersecret-git") + proc = _spawn_confined( + [ + "python3", + "-c", + "from pathlib import Path; print('ENV_BEGIN'); print(Path('/proc/1/environ').read_bytes().decode('utf-8','replace'))", + ], + cwd=tmp_path, + caller_env={}, + timeout_sec=30, + ) + text = proc.stdout or "" + expect_true(proc.returncode == 0 and "ENV_BEGIN" in text, (proc.returncode, proc.stderr)) + expect_false("leakme-parent-env" in text, text) + expect_false("GIT_CONFIG_VALUE_0" in text, text) + expect_false("supersecret-git" in text, text) + + +def test_confined_process_cannot_read_git_config(tmp_path: Path) -> None: + """`--ro-bind / /` makes the checkout config readable and this fails.""" + _skip_without_bwrap() + proc = _spawn_confined( + [ + "python3", + "-c", + "from pathlib import Path; p=Path('/agent/repos/mcproof/.git/config'); print('YES' if p.is_file() else 'NO')", + ], + cwd=tmp_path, + caller_env={}, + timeout_sec=30, + ) + expect_true(proc.returncode == 0, proc.stderr) + expect_true((proc.stdout or "").strip() == "NO", (proc.stdout, proc.stderr)) + + +def test_confined_process_cannot_connect_host_sockets(tmp_path: Path) -> None: + """A visible `/run` lets the sandbox connect to the host sockets and this fails.""" + _skip_without_bwrap() + script = ( + "import socket\n" + "socks=('/run/cursor/api.sock','/run/host-services/ssh-auth.sock')\n" + "ok=[]\n" + "for sock in socks:\n" + " h=socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n" + " h.settimeout(0.3)\n" + " try:\n" + " h.connect(sock)\n" + " ok.append(sock)\n" + " except OSError:\n" + " pass\n" + " finally:\n" + " h.close()\n" + "print('CONNECT_OK' if ok else 'NO')\n" + ) + proc = _spawn_confined(["python3", "-c", script], cwd=tmp_path, caller_env={}, timeout_sec=30) + expect_true(proc.returncode == 0, proc.stderr) + expect_true((proc.stdout or "").strip() == "NO", (proc.stdout, proc.stderr)) + + +def test_confined_process_has_resource_limits(tmp_path: Path) -> None: + """Removing `prlimit` leaves the address space unlimited and this fails.""" + _skip_without_bwrap() + script = ( + "import resource\n" + f"print(resource.getrlimit(resource.RLIMIT_AS)[0])\n" + f"print(resource.getrlimit(resource.RLIMIT_CPU)[0])\n" + ) + proc = _spawn_confined(["python3", "-c", script], cwd=tmp_path, caller_env={}, timeout_sec=30) + lines = [ln.strip() for ln in (proc.stdout or "").splitlines() if ln.strip()] + expect_true(len(lines) == 2, proc.stdout + (proc.stderr or "")) + memtotal = 0 + for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines(): + if line.startswith("MemTotal:"): + memtotal = int(line.split()[1]) * 1024 + soft = int(lines[0]) + # A fixed 32 GiB cap is above this host. Unlimited is -1, and -1 <= MemTotal. + expect_true(memtotal > 0 and soft > 0 and soft <= memtotal, (soft, memtotal)) + expect_true(soft == _address_space_cap_bytes(), (soft, _address_space_cap_bytes())) + expect_true(int(lines[1]) == _SANDBOX_CPU_SEC, lines) + + +def test_split_attribute_init_is_a_lint_finding() -> None: + """A column-zero regex misses both of these. Restoring it makes this fail.""" + samples = ( + "theorem t (n : Nat) : n = n := rfl attribute [init] pwnInit\n", + "theorem t (n : Nat) : n = n := rfl\nattribute\n[init] pwnInit\n", + ) + for src in samples: + if has_dep("soko"): + rules = {f["rule"] for f in lint_lean4_source(src, require_theorem=False)["findings"]} + expect_in("FORBIDDEN_INIT_ATTRIBUTE", rules, src) + side = {f["rule"] for f in _family_c_build_side_effect_findings(src)} + expect_in("FORBIDDEN_BUILD_SIDE_EFFECT", side, src) + + +def test_family_c_split_attribute_init_is_rejected() -> None: + """The full turn used to accept these with no rules. A line-anchored regex does that again.""" + _skip_without_confinement() + samples = ( + "theorem CrtPair.Spec.crt_mod_left (n : Nat) : n = n := rfl attribute [init] pwnInit\n" + "def pwnInit : IO Unit := pure ()\n", + "attribute\n[init] pwnInit\ndef pwnInit : IO Unit := pure ()\n", + ) + for snippet in samples: + findings = _run_family_c_real_lean_turn( + _family_c_refl_harness(), + _family_c_with_theorems(snippet), + ) + rules = {f["rule"] for f in findings} + expect_true( + "FORBIDDEN_BUILD_SIDE_EFFECT" in rules or "FORBIDDEN_INIT_ATTRIBUTE" in rules, + findings, + ) + + +def test_family_c_restores_proofs_and_comparator_after_the_build() -> None: + """The build rewrites both files. Skipping the restore drops the axiom finding or raises.""" + _skip_without_confinement() + _skip_without_soko() + snippet = ( + "axiom planted_bad : False\n" + "run_meta do\n" + " IO.FS.writeFile \"CrtPair/Proofs.lean\" " + "\"import CrtPair\\n\\ntheorem CrtPair.Spec.crt_mod_left (n : Nat) : n = n := rfl\\n" + "theorem CrtPair.Spec.crt_mod_right (n : Nat) : n = n := rfl\\n\"\n" + " IO.FS.writeFile \"comparator.json\" \"not-json\"\n" + ) + findings = _run_family_c_real_lean_turn( + _family_c_refl_harness(), + _family_c_with_theorems(snippet, prelude="import Lean\n"), + ) + rules = {f["rule"] for f in findings} + expect_in("EXTRA_AXIOM_DECL", rules, str(findings)) + expect_false("COMPARATOR_JSON_MALFORMED" in rules, findings) + + +def test_malformed_comparator_json_is_a_finding(tmp_path: Path) -> None: + """An uncaught `json.loads` raises `JSONDecodeError` and this errors instead of failing closed.""" + (tmp_path / "comparator.json").write_text("{", encoding="utf-8") + res = verify_lean4_project_comparator(tmp_path) + expect_false(res.passed) + rules = {f["rule"] for f in res.findings} + if _nofollow_dir_ops_available(): + expect_in("COMPARATOR_JSON_MALFORMED", rules) + else: + expect_in(_NOFOLLOW_OPEN_RULE, rules) + + +_ORIGINAL_PROOFS = "import Probe\ntheorem Probe.Proofs.goal : True := trivial\n" + + +def _write_toml_project(root: Path, proofs: str) -> None: + (root / "Probe").mkdir(parents=True) + (root / "lean-toolchain").write_text("leanprover/lean4:v4.35.0\n", encoding="utf-8") + (root / "lakefile.toml").write_text( + 'name = "probe"\nversion = "0.1.0"\n' + 'defaultTargets = ["Probe", "Probe.Proofs"]\n\n' + '[[lean_lib]]\nname = "Probe"\n\n[[lean_lib]]\nname = "Probe.Proofs"\n', + encoding="utf-8", + ) + (root / "Probe.lean").write_text("theorem probeRoot : True := trivial\n", encoding="utf-8") + (root / "Probe" / "Proofs.lean").write_text(proofs, encoding="utf-8") + (root / "comparator.json").write_text( + '{"challenge_module":"Probe","solution_module":"Probe.Proofs",' + '"theorem_names":["Probe.Proofs.goal"],' + '"permitted_axioms":["propext","Quot.sound","Classical.choice"]}\n', + encoding="utf-8", + ) + + +def test_planted_lakefile_is_not_elaborated(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Lake prefers lakefile.lean. Copying it into the trusted tree elaborates the plant.""" + _skip_without_confinement() + _skip_without_soko() + import mcproof.lean_harness as harness + + root = tmp_path / "proj" + _write_toml_project(root, _ORIGINAL_PROOFS) + marker = root / "plant-marker.txt" + (root / "lakefile.lean").write_text( + "import Lean\n" + "elab \"pwnPlant\" : command => do\n" + " let _ ← IO.FS.writeFile \"plant-marker.txt\" \"ELABORATED\"\n" + "pwnPlant\n", + encoding="utf-8", + ) + real = harness.run_confined_command + elaborated: list[str] = [] + + def _wrap(argv, *args, **kwargs): + res = real(argv, *args, **kwargs) + if any(str(part) == "build" for part in argv): + cwd = Path(kwargs.get("cwd") or ".") + if (cwd / "lakefile.lean").exists() or (cwd / "plant-marker.txt").is_file(): + elaborated.append(str(cwd)) + return res + + monkeypatch.setattr(harness, "run_confined_command", _wrap) + verify_lean4_project_comparator(root, allow_unsandboxed=False) + expect_false(marker.is_file(), "the planted lakefile was elaborated on the caller tree") + expect_true(elaborated == [], elaborated) + expect_true( + (root / "Probe" / "Proofs.lean").read_text(encoding="utf-8") == _ORIGINAL_PROOFS, + (root / "Probe" / "Proofs.lean").read_text(encoding="utf-8"), + ) + + +def test_comparator_cannot_see_the_host(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """A host `lake comparator` re-elaborates a cold lakefile and this report shows the host.""" + _skip_without_confinement() + _skip_without_soko() + monkeypatch.setenv("MCPROOF_SANDBOX_SENTINEL", "leakme-parent-env") + root = tmp_path / "proj" + (root / "Probe").mkdir(parents=True) + (root / "lean-toolchain").write_text("leanprover/lean4:v4.35.0\n", encoding="utf-8") + (root / "lakefile.toml").write_text( + 'name = "probe"\nversion = "0.1.0"\n' + 'defaultTargets = ["Probe", "Probe.Proofs"]\n\n' + '[[lean_lib]]\nname = "Probe"\n\n[[lean_lib]]\nname = "Probe.Proofs"\n', + encoding="utf-8", + ) + (root / "Probe.lean").write_text( + "import Lean\n" + "elab \"pwnProbe\" : command => do\n" + " let _ ← IO.Process.output { cmd := \"python3\", args := #[\"probe.py\"] }\n" + "pwnProbe\n" + "theorem probeRoot : True := trivial\n", + encoding="utf-8", + ) + (root / "Probe" / "Proofs.lean").write_text(_ORIGINAL_PROOFS, encoding="utf-8") + (root / "comparator.json").write_text( + '{"challenge_module":"Probe","solution_module":"Probe.Proofs",' + '"theorem_names":["Probe.Proofs.goal"],' + '"permitted_axioms":["propext","Quot.sound","Classical.choice"]}\n', + encoding="utf-8", + ) + hits: list[bytes] = [] + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + server.listen(1) + server.settimeout(0.2) + port = server.getsockname()[1] + stop = threading.Event() + + def serve() -> None: + while not stop.is_set(): + try: + conn, _addr = server.accept() + except TimeoutError: + continue + except OSError: + return + try: + hits.append(conn.recv(64)) + finally: + conn.close() + + thread = threading.Thread(target=serve, daemon=True) + thread.start() + (root / "probe.py").write_text( + "import os, socket\n" + "from pathlib import Path\n" + "phase = Path('probe-phase.txt')\n" + "tag = 'comparator' if phase.is_file() else 'build'\n" + "try:\n" + " if not phase.is_file():\n" + " phase.write_text('done\\n')\n" + "except OSError:\n" + " pass\n" + "git = Path('/agent/repos/mcproof/.git/config')\n" + "sock = '0'\n" + "try:\n" + " h = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n" + " h.settimeout(0.3)\n" + " h.connect('/run/host-services/ssh-auth.sock')\n" + " sock = '1'\n" + " h.close()\n" + "except OSError:\n" + " pass\n" + "tcp = '0'\n" + "try:\n" + f" c = socket.create_connection(('127.0.0.1', {port}), 0.3)\n" + " c.sendall(b'NET_OK')\n" + " c.close()\n" + " tcp = '1'\n" + "except OSError:\n" + " pass\n" + "env = os.environ.get('MCPROOF_SANDBOX_SENTINEL', '')\n" + "proc = Path('/proc/1/environ').read_bytes().decode('utf-8', 'replace') if Path('/proc/1/environ').is_file() else ''\n" + "sent = '1' if ('leakme-parent-env' in env or 'leakme-parent-env' in proc) else '0'\n" + "lake = Path('.lake')\n" + "lake.mkdir(exist_ok=True)\n" + "(lake / 'probe-report.txt').write_text(f'{tag} GIT={1 if git.is_file() else 0} SOCK={sock} TCP={tcp} SENTINEL={sent}\\n')\n" + "proofs = Path('Probe/Proofs.lean')\n" + "if tag == 'build':\n" + " proofs.write_text(proofs.read_text() + '-- rewritten-by-probe\\n')\n" + "else:\n" + " try:\n" + " proofs.write_text('ELABORATED\\n')\n" + " except OSError:\n" + " pass\n", + encoding="utf-8", + ) + import mcproof.lean_harness as harness + + real = harness.run_confined_command + reports: list[str] = [] + + def _wrap(argv, *args, **kwargs): + res = real(argv, *args, **kwargs) + if any(str(part) == "comparator" for part in argv): + report_path = Path(kwargs.get("cwd") or ".") / ".lake" / "probe-report.txt" + reports.append(report_path.read_text(encoding="utf-8") if report_path.is_file() else "") + return res + + monkeypatch.setattr(harness, "run_confined_command", _wrap) + try: + verify_lean4_project_comparator(root, allow_unsandboxed=False) + report = reports[-1] if reports else "" + expect_true(bool(reports), "the comparator command did not run") + expect_true("GIT=0" in report and "SOCK=0" in report and "TCP=0" in report, report) + expect_true("SENTINEL=0" in report and "leakme-parent-env" not in report, report) + expect_true(hits == [], hits) + expect_true( + (root / "Probe" / "Proofs.lean").read_text(encoding="utf-8") == _ORIGINAL_PROOFS, + (root / "Probe" / "Proofs.lean").read_text(encoding="utf-8"), + ) + finally: + stop.set() + server.close() + thread.join(timeout=2) + + +def test_eval_or_run_meta_never_runs_the_comparator(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Removing the early return reaches `lake comparator` and this fails.""" + _skip_without_soko() + if not has_lean_toolchain(): + pytest.skip("Lean is absent, so the comparator stop was not exercised") + + import mcproof.lean_harness as harness + + real_run = harness.subprocess.run + real_confined = harness.run_confined_command + + def reject_run(argv, *args, **kwargs): + if any(str(part) == "comparator" for part in argv): + raise AssertionError(f"comparator ran: {argv}") + return real_run(argv, *args, **kwargs) + + def reject_confined(argv, *args, **kwargs): + if any(str(part) == "comparator" for part in argv): + raise AssertionError(f"comparator ran: {argv}") + return real_confined(argv, *args, **kwargs) + + monkeypatch.setattr(harness, "run_confined_command", reject_confined) + monkeypatch.setattr(harness.subprocess, "run", reject_run) + samples = ( + _ORIGINAL_PROOFS + "#eval IO.println \"hi\"\n", + "import Lean\n" + _ORIGINAL_PROOFS + "run_meta do pure ()\n", + ) + for proofs in samples: + root = tmp_path / uuid.uuid4().hex + _write_toml_project(root, proofs) + res = verify_lean4_project_comparator(root, allow_unsandboxed=False) + rules = {f["rule"] for f in res.findings} + expect_true( + "FORBIDDEN_EVAL_COMMAND" in rules or any("run_meta" in str(f.get("message", "")) for f in res.findings), + res.findings, + ) + + +def test_confined_compile_error_reports_sandboxed() -> None: + """The compile ran inside the sandbox. Reporting sandboxed=False makes this fail.""" + _skip_without_confinement() + _skip_without_soko() + res = verify_lean4_source( + 'theorem t (n : Nat) : n = n := rfl\ndef s := r##"foo"#\n', + ) + expect_in("LEAN_COMPILE_ERROR", {f["rule"] for f in res.findings}) + expect_true(res.sandboxed is True, res) + + +class _Failed: + returncode = 1 + stdout = "" + stderr = "failed" + + +def _fingerprint(root: Path) -> tuple: + rows = [] + for path in sorted(root.rglob("*")): + rel = path.relative_to(root).as_posix() + if path.is_symlink(): + rows.append((rel, "symlink", str(path.readlink()))) + elif path.is_dir(): + rows.append((rel, "dir", "")) + else: + rows.append((rel, "file", path.read_bytes())) + return tuple(rows) + + +def _make_victim(path: Path) -> None: + path.mkdir() + (path / "file.txt").write_text("SENTINEL_FILE\n", encoding="utf-8") + (path / "dir.d").mkdir() + (path / "dir.d" / "secret").write_text("KEEP\n", encoding="utf-8") + (path / "CANARY").write_text("CANARY\n", encoding="utf-8") + + +def _plant_restore_targets(root: Path) -> None: + """Files a restore would copy. `dir.d` is a file here and a directory in the victim.""" + for name in ("Probe", "CrtPair", "notes"): + directory = root / name + directory.mkdir(exist_ok=True) + (directory / "file.txt").write_text(f"project-{name}-file\n", encoding="utf-8") + (directory / "dir.d").write_text(f"project-{name}-dir\n", encoding="utf-8") + + +def _swap_parents(root: Path, victims: dict[str, Path]) -> None: + for name, victim in victims.items(): + dest = root / name + if dest.is_symlink(): + continue + if dest.is_dir(): + shutil.rmtree(dest) + elif dest.exists(): + dest.unlink() + dest.symlink_to(victim, target_is_directory=True) + + +def _assert_victims_unchanged(victims: dict[str, Path], before: dict[str, tuple]) -> None: + for name, victim in victims.items(): + expect_true(_fingerprint(victim) == before[name], name) + expect_true((victim / "file.txt").read_text(encoding="utf-8") == "SENTINEL_FILE\n", name) + expect_true((victim / "dir.d" / "secret").read_text(encoding="utf-8") == "KEEP\n", name) + expect_true((victim / "CANARY").read_text(encoding="utf-8") == "CANARY\n", name) + expect_true(victim.is_dir() and not victim.is_symlink(), name) + + +def test_verify_parent_symlink_restore_does_not_touch_the_host( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A build that swaps Probe, CrtPair, or notes must not make the restore write the host.""" + _skip_without_confinement() + _skip_without_soko() + import mcproof.lean_harness as harness + + root = tmp_path / "proj" + _write_toml_project(root, _ORIGINAL_PROOFS) + _plant_restore_targets(root) + victims = {name: tmp_path / f"victim-{name}" for name in ("Probe", "CrtPair", "notes")} + for victim in victims.values(): + _make_victim(victim) + before = {name: _fingerprint(path) for name, path in victims.items()} + called = {"build": False} + + def _fake(argv, *args, **kwargs): + if any(str(part) == "build" for part in argv): + called["build"] = True + _swap_parents(root, victims) + return _Failed() + raise AssertionError(f"unexpected command: {argv}") + + monkeypatch.setattr(harness, "run_confined_command", _fake) + verify_lean4_project_comparator(root, allow_unsandboxed=False) + expect_true(called["build"], "the build never ran, so the symlink swap was not tested") + _assert_victims_unchanged(victims, before) + + +def test_family_c_parent_symlink_restore_does_not_touch_the_host( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Family C's failure path must not write the host through a swapped parent directory.""" + _skip_without_confinement() + import mcproof.lean_harness as harness + import mcproof.repair_loop as repair + + victims = {name: tmp_path / f"victim-{name}" for name in ("Probe", "CrtPair", "notes")} + for victim in victims.values(): + _make_victim(victim) + before = {name: _fingerprint(path) for name, path in victims.items()} + called = {"build": False} + + def _read(proj: Path): + _plant_restore_targets(Path(proj)) + return harness._read_project_tree(proj) + + def _fake(argv, *args, **kwargs): + if any(str(part) == "build" for part in argv): + called["build"] = True + _swap_parents(Path(kwargs["cwd"]), victims) + return _Failed() + raise AssertionError(f"unexpected command: {argv}") + + monkeypatch.setattr(repair, "_read_project_tree", _read) + monkeypatch.setattr(repair, "run_confined_command", _fake) + findings = _run_family_c_real_lean_turn(_family_c_refl_harness(), _family_c_with_theorems("")) + expect_true(called["build"], f"the build never ran: {findings}") + expect_in("LAKE_BUILD_FAILED", {f["rule"] for f in findings}) + _assert_victims_unchanged(victims, before) + + +def test_lake_symlink_is_a_finding(tmp_path: Path) -> None: + """`.lake` as a symlink is a finding. Following it, or rmtree on it, touches the target.""" + root = tmp_path / "proj" + _write_toml_project(root, _ORIGINAL_PROOFS) + precious = tmp_path / "precious" + precious.mkdir() + (precious / "KEEP").write_text("KEEP\n", encoding="utf-8") + lake = root / ".lake" + lake.symlink_to(precious, target_is_directory=True) + res = verify_lean4_project_comparator(root, allow_unsandboxed=True) + rules = {f["rule"] for f in res.findings} + if _nofollow_dir_ops_available(): + expect_in("LAKE_CACHE_SYMLINK", rules) + else: + expect_in(_NOFOLLOW_OPEN_RULE, rules) + expect_true((precious / "KEEP").read_text(encoding="utf-8") == "KEEP\n") + expect_true(lake.is_symlink()) + + +def test_family_c_lake_symlink_is_a_finding(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """The Family C build must not raise, follow, or delete through a `.lake` symlink.""" + import mcproof.repair_loop as repair + + precious = tmp_path / "precious" + precious.mkdir() + (precious / "KEEP").write_text("KEEP\n", encoding="utf-8") + + def _wrap(root: Path): + lake = Path(root) / ".lake" + if lake.is_symlink(): + lake.unlink() + elif lake.is_dir(): + shutil.rmtree(lake) + elif lake.exists(): + lake.unlink() + lake.symlink_to(precious, target_is_directory=True) + return repair_build(root) + + repair_build = repair._family_c_direct_lake_build + monkeypatch.setattr(repair, "_family_c_direct_lake_build", _wrap) + findings = _run_family_c_real_lean_turn(_family_c_refl_harness(), _family_c_with_theorems("")) + rules = {f["rule"] for f in findings} + if _nofollow_dir_ops_available(): + expect_in("LAKE_CACHE_SYMLINK", rules) + else: + expect_in(_NOFOLLOW_OPEN_RULE, rules) + expect_true((precious / "KEEP").read_text(encoding="utf-8") == "KEEP\n", findings) + expect_true((Path(precious)).is_dir(), findings) + + +def test_build_created_manifest_does_not_reach_the_comparator( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A manifest written by the build must not be the one `lake comparator` reads.""" + _skip_without_confinement() + _skip_without_soko() + import mcproof.lean_harness as harness + + root = tmp_path / "proj" + _write_toml_project(root, _ORIGINAL_PROOFS) + real = harness.run_confined_command + build_cwds: list[Path] = [] + seen: list[str] = [] + + def _wrap(argv, *args, **kwargs): + res = real(argv, *args, **kwargs) + cwd = Path(kwargs.get("cwd") or ".").resolve() + if any(str(part) == "build" for part in argv): + (cwd / "lake-manifest.json").write_text('{"evil-path-dep": true}\n', encoding="utf-8") + build_cwds.append(cwd) + if any(str(part) == "comparator" for part in argv): + manifest = cwd / "lake-manifest.json" + seen.append(manifest.read_text(encoding="utf-8") if manifest.is_file() else "") + expect_false(cwd in build_cwds, f"comparator reused the build directory {cwd}") + return res + + monkeypatch.setattr(harness, "run_confined_command", _wrap) + verify_lean4_project_comparator(root, allow_unsandboxed=False) + expect_true(bool(seen), "the comparator did not run") + expect_true(all("evil-path-dep" not in text for text in seen), seen) + + +def test_family_c_build_created_manifest_does_not_reach_the_comparator( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Family C's comparator copy is the pre-build tree, not the tree the build rewrote.""" + _skip_without_confinement() + _skip_without_soko() + import mcproof.lean_harness as harness + import mcproof.repair_loop as repair + + real = harness.run_confined_command + build_cwds: list[Path] = [] + seen: list[str] = [] + + def _wrap(argv, *args, **kwargs): + res = real(argv, *args, **kwargs) + cwd = Path(kwargs.get("cwd") or ".").resolve() + if any(str(part) == "build" for part in argv): + (cwd / "lake-manifest.json").write_text('{"evil-path-dep": true}\n', encoding="utf-8") + build_cwds.append(cwd) + if any(str(part) == "comparator" for part in argv): + manifest = cwd / "lake-manifest.json" + seen.append(manifest.read_text(encoding="utf-8") if manifest.is_file() else "") + expect_false(cwd in build_cwds, f"comparator reused the build directory {cwd}") + return res + + real_verify = repair.verify_lean4_project_comparator + entered: list[str] = [] + + def _verify(project_dir, *args, **kwargs): + manifest = Path(project_dir) / "lake-manifest.json" + entered.append(manifest.read_text(encoding="utf-8") if manifest.is_file() else "") + return real_verify(project_dir, *args, **kwargs) + + monkeypatch.setattr(harness, "run_confined_command", _wrap) + monkeypatch.setattr(repair, "run_confined_command", _wrap) + monkeypatch.setattr(repair, "verify_lean4_project_comparator", _verify) + _run_family_c_real_lean_turn(_family_c_refl_harness(), _family_c_with_theorems("")) + expect_true(bool(seen), "the comparator did not run") + expect_true(all("evil-path-dep" not in text for text in seen), seen) + expect_true(bool(entered), "verify was not given a trusted tree") + expect_true(all("evil-path-dep" not in text for text in entered), entered) + + +def test_allow_unsandboxed_does_not_run_lake_on_the_host( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A failed bwrap probe refuses the build. The flag must not run lake on the host.""" + _skip_without_soko() + import mcproof.lean_harness as harness + + monkeypatch.setattr(harness, "linux_bwrap_functional", lambda **_kwargs: False) + real_run = harness.subprocess.run + + def _host(argv, *args, **kwargs): + if any(part in {"lake", "lean"} or str(part).endswith(("/lake", "/lean")) for part in argv): + raise AssertionError(f"candidate command ran on the host: {argv}") + return real_run(argv, *args, **kwargs) + + def _confined(argv, *args, **kwargs): + raise AssertionError(f"candidate command ran without a working sandbox: {argv}") + + monkeypatch.setattr(harness.subprocess, "run", _host) + monkeypatch.setattr(harness, "run_confined_command", _confined) + root = tmp_path / "proj" + _write_toml_project(root, _ORIGINAL_PROOFS) + res = verify_lean4_project_comparator(root, allow_unsandboxed=True) + expect_false(res.passed) + expect_in("SANDBOX_UNAVAILABLE", {f["rule"] for f in res.findings}) + + +def test_run_cmd_and_hidden_eval_never_reach_lake(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """`run_cmd` and a `#eval` after a short raw-string closer never reach lake.""" + import mcproof.lean_harness as harness + import mcproof.repair_loop as repair + + real_run = harness.subprocess.run + + def _boom(argv, *args, **kwargs): + parts = [str(part) for part in argv] + if any(part == "lake" or part.endswith("/lake") or part == "lean" or part.endswith("/lean") for part in parts): + raise AssertionError(f"lake ran: {argv}") + if any(part in {"build", "comparator", "update"} for part in parts): + raise AssertionError(f"lake ran: {argv}") + return real_run(argv, *args, **kwargs) + + monkeypatch.setattr(harness, "run_confined_command", _boom) + monkeypatch.setattr(harness.subprocess, "run", _boom) + monkeypatch.setattr(repair, "run_confined_command", _boom) + hidden = 'def s := r##"foo"#\n#eval IO.FS.writeFile "/tmp/mcproof-should-not-run" "p"\n' + run_cmd = "run_cmd do pure ()\n" + for proofs in (_ORIGINAL_PROOFS + hidden, _ORIGINAL_PROOFS + run_cmd): + root = tmp_path / uuid.uuid4().hex + _write_toml_project(root, proofs) + res = verify_lean4_project_comparator(root, allow_unsandboxed=True) + rules = {f["rule"] for f in res.findings} + if _nofollow_dir_ops_available(): + expect_true( + "FORBIDDEN_EVAL_COMMAND" in rules or "METAPROGRAM_ESCAPE" in rules, + res.findings, + ) + else: + expect_in(_NOFOLLOW_OPEN_RULE, rules) + expect_false(res.passed) + for snippet in (hidden, run_cmd): + findings = _run_family_c_real_lean_turn( + _family_c_refl_harness(), + _family_c_with_theorems(snippet), + ) + rules = {f["rule"] for f in findings} + expect_true( + "FORBIDDEN_EVAL_COMMAND" in rules or "FORBIDDEN_ELAB_COMMAND" in rules, + findings, + ) + + +def test_missing_nofollow_open_refuses_without_copying(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Missing `O_DIRECTORY`, `O_NOFOLLOW`, or dir_fd support refuses. It does not follow a copy.""" + import os + + import mcproof.lean_harness as harness + import mcproof.repair_loop as repair + + def _swapped() -> tuple[Path, Path, tuple]: + root = tmp_path / uuid.uuid4().hex + _write_toml_project(root, _ORIGINAL_PROOFS) + victim = tmp_path / f"{root.name}-victim" + _make_victim(victim) + before = _fingerprint(victim) + probe = root / "Probe" + shutil.rmtree(probe) + probe.symlink_to(victim, target_is_directory=True) + return root, victim, before + + def _expect_refusal() -> None: + root, victim, before = _swapped() + res = verify_lean4_project_comparator(root, allow_unsandboxed=True) + expect_false(res.passed) + expect_in(_NOFOLLOW_OPEN_RULE, {f["rule"] for f in res.findings}) + raised = False + try: + harness._read_project_tree(root) + except UntrustedProjectPath as exc: + raised = exc.rule == _NOFOLLOW_OPEN_RULE + expect_true(raised, "the tree read followed a path or used another rule") + try: + harness._materialize_trusted_tree({"Probe/file.txt": b"SENTINEL_FILE\n"}) + except UntrustedProjectPath as exc: + raised = exc.rule == _NOFOLLOW_OPEN_RULE + else: + raised = False + expect_true(raised, "materialize copied without a non-following open") + expect_false(harness._real_dir(root, "Probe")) + findings = repair._family_c_direct_lake_build(root) + expect_in(_NOFOLLOW_OPEN_RULE, {f["rule"] for f in findings}) + expect_true(_fingerprint(victim) == before) + expect_true((victim / "file.txt").read_text(encoding="utf-8") == "SENTINEL_FILE\n") + expect_true((victim / "dir.d" / "secret").read_text(encoding="utf-8") == "KEEP\n") + + if _nofollow_dir_ops_available(): + with monkeypatch.context() as hidden: + hidden.delattr(os, "O_DIRECTORY", raising=False) + hidden.delattr(os, "O_NOFOLLOW", raising=False) + expect_false(_nofollow_dir_ops_available()) + _expect_refusal() + expect_true(_nofollow_dir_ops_available()) + with monkeypatch.context() as hidden: + hidden.setattr(os, "supports_dir_fd", frozenset()) + expect_false(_nofollow_dir_ops_available()) + _expect_refusal() + expect_true(_nofollow_dir_ops_available()) + else: + _expect_refusal() diff --git a/tests/test_domain_and_repair_loop.py b/tests/test_domain_and_repair_loop.py index 190e167..cf36300 100644 --- a/tests/test_domain_and_repair_loop.py +++ b/tests/test_domain_and_repair_loop.py @@ -2,11 +2,17 @@ from __future__ import annotations +import importlib.util import io import json from pathlib import Path +import re +import subprocess import sys from typing import Any, Mapping, Sequence +import uuid + +import pytest from mcproof.cli import main as cli_main from mcproof.domain_protocol import ( @@ -23,9 +29,17 @@ verify_minimal_goal_stopping, verify_rational_lp_certificate, ) +from mcproof.lean_harness import ( + _NOFOLLOW_OPEN_RULE, + _nofollow_dir_ops_available, + linux_bwrap_functional, + strip_lean_comments_and_strings, +) from mcproof.paths import repo_root from mcproof.repair_loop import ( MAX_RETRIEVAL_TOKENS_HARD_CAP, + _family_c_direct_lake_build, + _run_family_c_real_lean_turn, compute_candidate_state_hash, compute_failure_signature, default_inference_oracle, @@ -36,7 +50,7 @@ sanitize_oracle_diagnostics, ) -from helpers import expect_eq, expect_false, expect_in, expect_true, requires_dep, requires_mde +from helpers import expect_eq, expect_false, expect_in, expect_true, has_dep, has_lean_toolchain, requires_dep, requires_mde def test_targeted_plane1_retrieval_stays_under_1500_tokens_across_validation_split() -> None: @@ -457,3 +471,548 @@ def test_audit_transition_certificate_catches_f01_to_f08_and_a6_a7(tmp_path: Pat lp_farkas_ok = verify_rational_lp_certificate({"mode": "infeasible", "A": [[-1, -1]], "b": [1], "y": [1]}) expect_true(lp_farkas_ok["passed"]) + + +# Coprime two-modulus residue claims. These are the target statements the guard hashes. +# They are not `op x = x`. +_CRT_LEFT = ( + "(m1 m2 x y : Nat) (hcop : Nat.gcd m1 m2 = 1) " + "(h1 : Dvd.dvd m1 (x - y)) (h2 : Dvd.dvd m2 (x - y)) : Dvd.dvd (m1 * m2) (x - y)" +) +_CRT_RIGHT = ( + "(m1 m2 x : Nat) (hcop : Nat.gcd m1 m2 = 1) (hx : Dvd.dvd (m1 * m2) x) : Dvd.dvd m1 x" +) + + +def _family_c_spoof_harness() -> dict[str, Any]: + return { + "oracle_type": "bar_check", + "spec_meta": { + "pkg": "CrtPair", + "bar_status": "in-progress", + "test_cmd": "python3 tests/run_tests.py", + "theorem_names": ["CrtPair.Spec.crt_mod_left", "CrtPair.Spec.crt_mod_right"], + "statements": { + "CrtPair.Spec.crt_mod_left": _CRT_LEFT, + "CrtPair.Spec.crt_mod_right": _CRT_RIGHT, + }, + }, + } + + +def test_family_c_real_lean_turn_rejects_says_verified_spoof() -> None: + comp = "Claim-Tier: PROVEN\n\nStatus: verified. All checks passed.\n" + findings = _run_family_c_real_lean_turn(_family_c_spoof_harness(), comp) + expect_true(bool(findings), "spoofed verified claim must fail the turn") + expect_in("FAMILY_C_STATUS_SELF_ATTESTATION", {f["rule"] for f in findings}) + + +def test_family_c_identity_plus_rfl_scaffold_no_longer_passes() -> None: + comp = ( + "Claim-Tier: PROVEN\n\nStatus: verified.\n\n" + "```lean\ntheorem crt_mod_left (x : Nat) : CrtPair.op x = x := rfl\n```\n" + ) + findings = _run_family_c_real_lean_turn(_family_c_spoof_harness(), comp) + expect_true(bool(findings), "identity scaffold with rfl must fail the turn") + expect_in("FAMILY_C_THEOREM_NAME_MISMATCH", {f["rule"] for f in findings}) + + +def test_family_c_lean_error_text_reaches_next_turn_prompt() -> None: + """Formats a hand-written diagnostic. This test does not run `lake`.""" + findings = [ + { + "line": 1, + "rule": "LAKE_BUILD_FAILED", + "message": "`lake build` failed: error: CrtPair/Proofs.lean:2:59: Unknown constant `Nat.noSuchLemma` | error: build failed", + } + ] + text = format_diagnostics(findings, family="C") + expect_in("LAKE_BUILD_FAILED", text) + expect_in("Proofs.lean:2:59", text) + + +def test_family_c_fully_qualified_identity_rfl_fails_statement_hash() -> None: + """Fully-qualified `rfl` of `op x = x` must fail the pinned-statement hash. + + The names match the challenge, so a disabled hash compare + (`if False and cand[name] != exp[name]`) lets this candidate through the guard + and this test fails. + """ + comp = ( + "Status: in progress.\n\n" + "```lean\n" + "theorem CrtPair.Spec.crt_mod_left (x : Nat) : CrtPair.op x = x := rfl\n" + "theorem CrtPair.Spec.crt_mod_right (x : Nat) : CrtPair.op x = x := rfl\n" + "```\n" + ) + findings = _run_family_c_real_lean_turn(_family_c_spoof_harness(), comp) + rules = {f["rule"] for f in findings} + expect_in("FAMILY_C_STATEMENT_MISMATCH", rules) + expect_false("LAKE_BUILD_FAILED" in rules, "a statement mismatch must return before lake") + expect_false("FAMILY_C_THEOREM_NAME_MISMATCH" in rules, rules) + + +def test_family_c_missing_statement_pin_does_not_fall_back_to_identity() -> None: + harness = _family_c_spoof_harness() + del harness["spec_meta"]["statements"] + comp = ( + "```lean\n" + "theorem CrtPair.Spec.crt_mod_left (x : Nat) : CrtPair.op x = x := rfl\n" + "theorem CrtPair.Spec.crt_mod_right (x : Nat) : CrtPair.op x = x := rfl\n" + "```\n" + ) + findings = _run_family_c_real_lean_turn(harness, comp) + expect_in("FAMILY_C_TARGET_STATEMENT_ABSENT", {f["rule"] for f in findings}) + + +def test_family_c_known_bad_proof_reaches_lake_build() -> None: + """Run `lake build` on a statement-exact candidate whose proof does not elaborate. + + Skips only when `lean` and `lake` are absent. The skip reason says the lake path was not exercised. + """ + if not has_lean_toolchain(): + pytest.skip( + "Lean 4.35.0 (`lean` and `lake`) is absent, so the Family C lake path was not exercised" + ) + comp = ( + "Status: in progress.\n\n" + "```lean\n" + f"theorem CrtPair.Spec.crt_mod_left {_CRT_LEFT} := by\n exact Nat.noSuchLemma x\n" + f"theorem CrtPair.Spec.crt_mod_right {_CRT_RIGHT} := by\n exact Nat.noSuchLemma x\n" + "```\n" + ) + findings = _run_family_c_real_lean_turn(_family_c_spoof_harness(), comp) + rules = {f["rule"] for f in findings} + expect_in("LAKE_BUILD_FAILED", rules) + blob = "\n".join(str(f.get("message", "")) for f in findings) + expect_in("noSuchLemma", blob) + expect_true(bool(re.search(r"\.lean:\d+:\d+", blob)), blob) + + +_REFL_HEADER = "(n : Nat) : n = n" + + +def _family_c_refl_harness() -> dict[str, Any]: + return { + "oracle_type": "bar_check", + "spec_meta": { + "pkg": "CrtPair", + "bar_status": "in-progress", + "test_cmd": "python3 tests/run_tests.py", + "theorem_names": ["CrtPair.Spec.crt_mod_left", "CrtPair.Spec.crt_mod_right"], + "statements": { + "CrtPair.Spec.crt_mod_left": _REFL_HEADER, + "CrtPair.Spec.crt_mod_right": _REFL_HEADER, + }, + }, + } + + +def _family_c_refl_completion() -> str: + readme = (repo_root() / "datasets" / "targets" / "family_c" / "PUB-C-V01.md").read_text(encoding="utf-8") + return ( + readme + + "\n```lean\n" + + "theorem CrtPair.Spec.crt_mod_left (n : Nat) : n = n := rfl\n" + + "theorem CrtPair.Spec.crt_mod_right (n : Nat) : n = n := rfl\n" + + "```\n" + ) + + +def test_family_c_correct_proof_passes_when_manifest_is_kept() -> None: + """A correct `rfl` proof yields no findings when the template manifest is kept. + + `bar/check.py` item 3 fails closed without `lake-manifest.json`, including under + `--no-lean`. Deleting that file makes this test fail with `BAR_CHECK_FAILED`. + The test skips only when `lean` or the pinned `mission` checkout is absent. + """ + if not has_lean_toolchain(): + pytest.skip( + "Lean 4.35.0 (`lean` and `lake`) is absent, so the Family C correct-proof path was not exercised" + ) + if not has_dep("mission"): + pytest.skip( + "pinned dependency 'mission' is absent, so the Family C bar path was not exercised" + ) + findings = _run_family_c_real_lean_turn(_family_c_refl_harness(), _family_c_refl_completion()) + expect_eq(findings, [], "a correct proof must not be rejected when lake-manifest.json is kept") + + +def test_family_c_midline_theorem_counts_as_an_extra_name() -> None: + """A theorem that does not start a line is still part of the name set.""" + comp = ( + "```lean\n" + "theorem CrtPair.Spec.crt_mod_left (n : Nat) : n = n := rfl\n" + "theorem CrtPair.Spec.crt_mod_right (n : Nat) : n = n := rfl\n" + "def pad := 0 theorem Extra : True := trivial\n" + "```\n" + ) + findings = _run_family_c_real_lean_turn(_family_c_refl_harness(), comp) + rules = {f["rule"] for f in findings} + expect_in("FAMILY_C_THEOREM_NAME_MISMATCH", rules) + expect_true(any("Extra" in str(f.get("message", "")) for f in findings), findings) + expect_false("LAKE_BUILD_FAILED" in rules, findings) + expect_false("BAR_CHECK_FAILED" in rules, findings) + + +def _family_c_block_lake(monkeypatch: pytest.MonkeyPatch) -> dict[str, bool]: + """Record a lake or comparator call and return no findings from either.""" + called = {"lake": False} + + def _no_lake(*_args: object, **_kwargs: object) -> list[dict[str, Any]]: + called["lake"] = True + return [] + + def _no_cmp(*_args: object, **_kwargs: object) -> Any: + called["lake"] = True + return type("Cmp", (), {"findings": ()})() + + monkeypatch.setattr("mcproof.repair_loop._family_c_direct_lake_build", _no_lake) + monkeypatch.setattr("mcproof.repair_loop.verify_lean4_project_comparator", _no_cmp) + return called + + +def test_family_c_midline_eval_commands_rejected_before_lake(monkeypatch: pytest.MonkeyPatch) -> None: + """Every command in the eval lint is rejected before `lake`, including mid-line. + + The turn reads `FORBIDDEN_EVAL_COMMAND` itself. Putting the start-of-line + anchor back on that pattern lets these completions reach lake and this test fails. + A `#eval` or `#reduce` that sits in a comment or a string does not. + """ + called = _family_c_block_lake(monkeypatch) + for command in ("#eval", "#reduce", "#check", "#print", "#synth"): + called["lake"] = False + comp = ( + "```lean\n" + "theorem CrtPair.Spec.crt_mod_left (n : Nat) : n = n := rfl\n" + "theorem CrtPair.Spec.crt_mod_right (n : Nat) : n = n := rfl\n" + f"def pad := 0 {command} (1 : Nat)\n" + "```\n" + ) + findings = _run_family_c_real_lean_turn(_family_c_refl_harness(), comp) + rules = {f["rule"] for f in findings} + expect_false(called["lake"], f"lake ran for {command}: {findings}") + expect_in("FORBIDDEN_EVAL_COMMAND", rules, command) + expect_true(any(command in str(f.get("message", "")) for f in findings), findings) + expect_false("LAKE_BUILD_FAILED" in rules, findings) + decoy = ( + "```lean\n" + "theorem CrtPair.Spec.crt_mod_left (n : Nat) : n = n := rfl\n" + "theorem CrtPair.Spec.crt_mod_right (n : Nat) : n = n := rfl\n" + "-- #eval IO.println \"x\"\n" + "def note := \"#reduce\"\n" + "```\n" + ) + called["lake"] = False + decoy_findings = _run_family_c_real_lean_turn(_family_c_refl_harness(), decoy) + decoy_rules = {f["rule"] for f in decoy_findings} + expect_false("FORBIDDEN_EVAL_COMMAND" in decoy_rules, decoy_findings) + if not _nofollow_dir_ops_available(): + expect_in(_NOFOLLOW_OPEN_RULE, decoy_rules) + expect_false(called["lake"], decoy_findings) + + +def test_family_c_raw_string_and_char_literal_do_not_hide_eval( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A real `#eval` after a raw string or a character literal is rejected before lake. + + `r"\\"`, `r#"\\"#, `r###"\\"###`, `'"'`, and `'\\"'` used to desync the stripper, + so the command vanished from the pre-lake check and from the comparator lint. + Reverting the lexer makes `stripped` lose `#eval` and lets lake run. + """ + called = _family_c_block_lake(monkeypatch) + snippets = ( + 'def s := r"\\"\n#eval (1 : Nat)\n', + 'def s := r#"\\"#\n#eval (1 : Nat)\n', + 'def s := r###"\\"###\n#eval (1 : Nat)\n', + "def c := '\"'\n#eval (1 : Nat)\n", + "def c := '\\\"'\n#eval (1 : Nat)\n", + ) + for snippet in snippets: + called["lake"] = False + stripped = strip_lean_comments_and_strings(snippet) + expect_in("#eval", stripped, snippet) + findings = _run_family_c_real_lean_turn(_family_c_refl_harness(), _family_c_with_theorems(snippet)) + rules = {f["rule"] for f in findings} + expect_false(called["lake"], f"lake ran for {snippet!r}: {findings}") + expect_in("FORBIDDEN_EVAL_COMMAND", rules, snippet) + expect_true(any("#eval" in str(f.get("message", "")) for f in findings), findings) + expect_false("LAKE_BUILD_FAILED" in rules, findings) + + +def test_family_c_initializers_rejected_before_lake(monkeypatch: pytest.MonkeyPatch) -> None: + """`initialize`, `@[init]`, `builtin_initialize`, and `@[builtin_init]` never reach lake.""" + called = _family_c_block_lake(monkeypatch) + cases = ( + ("initialize do pure ()\n", "initialize"), + ("@[init] def pwnInit : IO Unit := pure ()\n", "@[init"), + ("builtin_initialize do pure ()\n", "builtin_initialize"), + ("@[builtin_init] def pwnBuilt : IO Unit := pure ()\n", "@[builtin_init"), + ("@[inline, init] def pwnInline : IO Unit := pure ()\n", "init"), + ) + for snippet, needle in cases: + called["lake"] = False + findings = _run_family_c_real_lean_turn(_family_c_refl_harness(), _family_c_with_theorems(snippet)) + rules = {f["rule"] for f in findings} + expect_false(called["lake"], f"lake ran for {needle}: {findings}") + expect_in("FORBIDDEN_INITIALIZER", rules, needle) + expect_true(any(needle in str(f.get("message", "")) for f in findings), findings) + expect_false("LAKE_BUILD_FAILED" in rules, findings) + + +def test_family_c_elab_commands_rejected_before_lake(monkeypatch: pytest.MonkeyPatch) -> None: + """`run_cmd`, `run_elab`, `run_tac`, and `elab` are rejected before the first lake build.""" + called = _family_c_block_lake(monkeypatch) + cases = ( + ("run_cmd pure ()\n", "run_cmd"), + ("run_elab pure ()\n", "run_elab"), + ("run_tac pure ()\n", "run_tac"), + ('elab "pwn" : command => pure ()\n', "elab"), + ("def pad := 0\nrun_cmd pure ()\n", "run_cmd"), + ) + for snippet, needle in cases: + called["lake"] = False + findings = _run_family_c_real_lean_turn(_family_c_refl_harness(), _family_c_with_theorems(snippet)) + rules = {f["rule"] for f in findings} + expect_false(called["lake"], f"lake ran for {needle}: {findings}") + expect_in("FORBIDDEN_ELAB_COMMAND", rules, needle) + expect_true(any(needle in str(f.get("message", "")) for f in findings), findings) + expect_false("LAKE_BUILD_FAILED" in rules, findings) + + +def test_family_c_banned_commands_inside_text_are_not_findings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Comments, docstrings, strings, and README prose are not elaboration commands.""" + called = _family_c_block_lake(monkeypatch) + comp = ( + "The README mentions #eval, initialize, @[init], builtin_initialize, " + "run_cmd, run_elab, run_tac, and elab.\n\n" + "```lean\n" + "theorem CrtPair.Spec.crt_mod_left (n : Nat) : n = n := rfl\n" + "theorem CrtPair.Spec.crt_mod_right (n : Nat) : n = n := rfl\n" + "-- #eval initialize @[init] builtin_initialize run_cmd run_elab run_tac elab\n" + "/-- #eval initialize @[init] builtin_initialize run_cmd run_elab run_tac elab -/\n" + 'def note := "#eval initialize @[init] builtin_initialize run_cmd run_elab run_tac elab"\n' + 'def raw := r"#eval initialize @[init] run_cmd run_elab run_tac elab"\n' + "def ch := '\"'\n" + "```\n" + ) + findings = _run_family_c_real_lean_turn(_family_c_refl_harness(), comp) + rules = {f["rule"] for f in findings} + expect_false("FORBIDDEN_EVAL_COMMAND" in rules, findings) + expect_false("FORBIDDEN_INITIALIZER" in rules, findings) + expect_false("FORBIDDEN_ELAB_COMMAND" in rules, findings) + if not _nofollow_dir_ops_available(): + expect_in(_NOFOLLOW_OPEN_RULE, rules) + expect_false(called["lake"], findings) + + +def _family_c_with_theorems(snippet: str, *, prelude: str = "") -> str: + return ( + "```lean\n" + + prelude + + "theorem CrtPair.Spec.crt_mod_left (n : Nat) : n = n := rfl\n" + + "theorem CrtPair.Spec.crt_mod_right (n : Nat) : n = n := rfl\n" + + snippet + + "```\n" + ) + + +def _skip_without_family_c_sandbox() -> None: + if not has_lean_toolchain(): + pytest.skip( + "Lean 4.35.0 (`lean` and `lake`) is absent, so the Family C sandbox path was not exercised" + ) + if not linux_bwrap_functional(): + pytest.skip( + "Linux bwrap is missing or non-functional, so the Family C sandbox path was not exercised" + ) + + +def _expect_family_c_marker_absent(snippet: str, *, prelude: str = "") -> None: + """The host marker must stay absent. A passthrough sandbox writes it and this fails.""" + marker = Path("/tmp") / f"mcproof-sandbox-marker-{uuid.uuid4().hex}" + marker.unlink(missing_ok=True) + body = snippet.replace("MARKER", str(marker)) + try: + _run_family_c_real_lean_turn( + _family_c_refl_harness(), + _family_c_with_theorems(body, prelude=prelude), + ) + expect_false( + marker.is_file(), + f"candidate wrote {marker}; the sandbox did not confine the build", + ) + finally: + marker.unlink(missing_ok=True) + + +def test_family_c_attribute_init_cannot_write_a_host_marker() -> None: + """`attribute [init]` runs when the axiom driver imports the built module. + + The pre-lake regex only matches `@[`, so it does not keep this marker absent. + Disabling the sandbox lets the import write the host file and this test fails. + """ + _skip_without_family_c_sandbox() + _expect_family_c_marker_absent( + 'def pwnInit : IO Unit := IO.FS.writeFile "MARKER" "p"\nattribute [init] pwnInit\n' + ) + + +def test_family_c_short_raw_string_cannot_write_a_host_marker() -> None: + """A raw string closed with fewer hashes hides `#eval` from the stripper. + + The turn rejects that command before lake, so the marker stays absent. + Reaching the build makes the spy in the confinement test fail. + """ + _skip_without_family_c_sandbox() + _expect_family_c_marker_absent( + 'def s := r##"foo"#\n#eval IO.FS.writeFile "MARKER" "p"\n' + ) + + +def test_family_c_lambda_and_sigma_raw_string_cannot_write_a_host_marker() -> None: + """`λ` and `Σ` hide a raw string from the stripper. Lean still runs the `#eval`.""" + _skip_without_family_c_sandbox() + for snippet in ( + 'def s := λr"\\"\n#eval IO.FS.writeFile "MARKER" "p"\n', + 'def s := Σr"\\"\n#eval IO.FS.writeFile "MARKER" "p"\n', + ): + _expect_family_c_marker_absent(snippet) + + +def test_family_c_unclosed_char_literal_cannot_write_a_host_marker() -> None: + """An unclosed `'\"` hides the next `#eval`. Lean reports the error and still runs it.""" + _skip_without_family_c_sandbox() + _expect_family_c_marker_absent( + 'def c := \'"\n#eval IO.FS.writeFile "MARKER" "p"\n' + ) + + +def test_family_c_interpolation_cannot_write_a_host_marker() -> None: + """`s!\"{\"}\"` hides the next `#eval`. Lean still runs it during the lake build.""" + _skip_without_family_c_sandbox() + _expect_family_c_marker_absent( + 'def s := s!"{"}"\n#eval IO.FS.writeFile "MARKER" "p"\n' + ) + + +def test_family_c_run_meta_cannot_write_a_host_marker() -> None: + """`run_meta` runs in the first lake build, before the comparator flags it.""" + _skip_without_family_c_sandbox() + _expect_family_c_marker_absent( + 'run_meta do IO.FS.writeFile "MARKER" "p"\n', + prelude="import Lean\n", + ) + + +def test_family_c_elab_rules_cannot_write_a_host_marker() -> None: + """`elab_rules` runs in the first lake build, before the comparator flags it.""" + _skip_without_family_c_sandbox() + _expect_family_c_marker_absent( + 'syntax "pwn" : command\n' + "elab_rules : command\n" + ' | `(command| pwn) => do IO.FS.writeFile "MARKER" "p"\n' + "pwn\n", + prelude="import Lean\n", + ) + + +def test_family_c_refuses_candidate_build_when_sandbox_is_unavailable( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A candidate build is refused when `bwrap` is unavailable. This needs neither Lean nor bwrap.""" + monkeypatch.setattr("mcproof.repair_loop.linux_bwrap_functional", lambda **_kwargs: False) + monkeypatch.setattr("mcproof.lean_harness.linux_bwrap_functional", lambda **_kwargs: False) + + def _refuse(cmd: object, *_args: object, **_kwargs: object) -> None: + raise AssertionError(f"candidate command ran without a sandbox: {cmd}") + + monkeypatch.setattr(subprocess, "run", _refuse) + findings = _family_c_direct_lake_build(tmp_path) + rules = {f["rule"] for f in findings} + if _nofollow_dir_ops_available(): + expect_eq(rules, {"SANDBOX_UNAVAILABLE"}, str(findings)) + expect_true(any("unsandboxed" in str(f.get("message", "")) for f in findings), findings) + else: + expect_eq(rules, {_NOFOLLOW_OPEN_RULE}, str(findings)) + + +def test_family_c_compilable_elab_names_are_not_commands(monkeypatch: pytest.MonkeyPatch) -> None: + """`elab'`, `x.elab`, and `«elab»` compile. The bare-word ban must not reject them.""" + called = _family_c_block_lake(monkeypatch) + for snippet in ("def elab' := 1\n", "def x.elab := 1\n", "def «elab» := 1\n", "def elab.foo := 1\n"): + called["lake"] = False + findings = _run_family_c_real_lean_turn(_family_c_refl_harness(), _family_c_with_theorems(snippet)) + rules = {f["rule"] for f in findings} + expect_false("FORBIDDEN_ELAB_COMMAND" in rules, f"{snippet!r}: {findings}") + if _nofollow_dir_ops_available(): + expect_true(called["lake"], f"{snippet!r} never reached the build: {findings}") + else: + expect_in(_NOFOLLOW_OPEN_RULE, rules) + expect_false(called["lake"], findings) + + +def test_family_c_non_ascii_and_other_decls_count_as_extra_names( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`«Extra»`, `α`, `café`, `instance`, `example`, and `abbrev` are extra names.""" + called = _family_c_block_lake(monkeypatch) + cases = ( + ("theorem «Extra» : True := trivial\n", "«Extra»"), + ("def pad := 0 theorem «Extra» : True := trivial\n", "«Extra»"), + ("theorem α : True := trivial\n", "α"), + ("def pad := 0 theorem α : True := trivial\n", "α"), + ("theorem café : True := trivial\n", "café"), + ("instance extraInst : Inhabited Unit := ⟨()⟩\n", "extraInst"), + ("def pad := 0 instance extraInst : Inhabited Unit := ⟨()⟩\n", "extraInst"), + ("example : True := trivial\n", "example"), + ("abbrev FooN := Nat\n", "FooN"), + ) + for snippet, needle in cases: + called["lake"] = False + comp = ( + "```lean\n" + "theorem CrtPair.Spec.crt_mod_left (n : Nat) : n = n := rfl\n" + "theorem CrtPair.Spec.crt_mod_right (n : Nat) : n = n := rfl\n" + + snippet + + "```\n" + ) + findings = _run_family_c_real_lean_turn(_family_c_refl_harness(), comp) + rules = {f["rule"] for f in findings} + expect_false(called["lake"], f"lake ran for {needle}: {findings}") + expect_in("FAMILY_C_THEOREM_NAME_MISMATCH", rules, needle) + expect_true(any(needle in str(f.get("message", "")) for f in findings), findings) + expect_false("LAKE_BUILD_FAILED" in rules, findings) + + +def test_grade_item_pins_statements_and_refuses_identity_scaffold() -> None: + script = repo_root() / "datasets" / "scripts" / "grade_item.py" + spec = importlib.util.spec_from_file_location("grade_item_under_test", script) + expect_true(spec is not None and spec.loader is not None, "grade_item.py failed to load") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) # type: ignore[union-attr] + harness = _family_c_spoof_harness() + bare = json.loads(json.dumps(harness)) + del bare["spec_meta"]["statements"] + ok, errs = mod._run_family_c_structural_replay(bare, "Status: in progress.\n") + expect_false(ok, errs) + expect_true(any("target_statements_absent" in err for err in errs), errs) + readme = (repo_root() / "datasets" / "targets" / "family_c" / "PUB-C-V01.md").read_text(encoding="utf-8") + ok_pinned, pinned_errs = mod._run_family_c_structural_replay(harness, readme) + mission_check = mod.DEPS_ROOT / "mission" / "bar" / "check.py" + if not mission_check.is_file(): + expect_false(ok_pinned, pinned_errs) + expect_true(any("missing_mission_dep" in err for err in pinned_errs), pinned_errs) + return + expect_true(ok_pinned, pinned_errs) + spec_src = mod._family_c_spec_source("CrtPair", harness["spec_meta"]["statements"]) + expect_in("Dvd.dvd (m1 * m2) (x - y)", spec_src) + expect_false("op x = x" in spec_src, spec_src) + lake = mod._family_c_lakefile("CrtPair") + expect_in('"CrtPair.Spec"', lake) + expect_in('"CrtPair.Proofs"', lake) + expect_false(lake.split("defaultTargets = ", 1)[1].startswith('["CrtPair"]\n'), lake) diff --git a/tests/test_lean_harness.py b/tests/test_lean_harness.py index 64d0787..9213bba 100644 --- a/tests/test_lean_harness.py +++ b/tests/test_lean_harness.py @@ -129,7 +129,7 @@ def test_reference_lean_project_passes_six_kernel_comparator() -> None: if not linux_bwrap_functional(): pytest.skip( "Linux bwrap is missing or non-functional on this host " - "(minimal `bwrap --ro-bind / / true` or lake `/run/user` session probe failed)" + "(the confinement probe rejected the sandbox)" ) res = verify_lean4_project_comparator(repo_root() / "lean", allow_unsandboxed=False) expect_true(res.passed, f"reference lean project failed: {res.findings}") diff --git a/tests/test_regex_fullmatch.py b/tests/test_regex_fullmatch.py index 063eb10..449464a 100644 --- a/tests/test_regex_fullmatch.py +++ b/tests/test_regex_fullmatch.py @@ -2,6 +2,8 @@ from __future__ import annotations +import importlib.machinery +import importlib.util import json from pathlib import Path @@ -59,3 +61,15 @@ def test_upstream_commit_trailing_newline_rejected(tmp_path: Path) -> None: (deps / "DEPS_MANIFEST.json").write_text(json.dumps({"repos": repos}), encoding="utf-8") with pytest.raises(StaleLineageError): build_lineage_manifest(root, d_dir=deps) + + +def test_bootstrap_deps_rejects_commit_pin_with_trailing_newline(tmp_path: Path) -> None: + script = repo_root() / "bin" / "bootstrap-deps" + loader = importlib.machinery.SourceFileLoader("bootstrap_deps_under_test", str(script)) + spec = importlib.util.spec_from_loader(loader.name, loader) + if spec is None: + raise AssertionError("bootstrap-deps failed to load") + mod = importlib.util.module_from_spec(spec) + loader.exec_module(mod) + with pytest.raises(ValueError): + mod._clone_at_exact_sha("https://example.invalid/repo.git", "a" * 40 + "\n", tmp_path / "dest") diff --git a/verification/VERIFICATION_RECEIPT.json b/verification/VERIFICATION_RECEIPT.json index 74c95fa..5e5aa78 100644 --- a/verification/VERIFICATION_RECEIPT.json +++ b/verification/VERIFICATION_RECEIPT.json @@ -591,7 +591,7 @@ "base_gguf_q8_0_sha256": "10cc391b403021dd11c614679d2fd92f611c3681d29e29651b717316965d61e1", "compiler_manifest_sha256": "fcec69eace974d2cb8629935864ced711a8859b07e7a396e530cf373f2606fac", "corpus_tree_sha256": "93624aae09eea99c3d35ebfa2e9acde1f9ac9c675167461652486850f206b642", - "datasets_sha256sums_manifest_sha256": "312953bc99bb42e6cbef1d1a3c240975f2baf3c60506df6c9eef06a678d45d56", + "datasets_sha256sums_manifest_sha256": "3e81ab327058baca6739841ea6c664d1b8a5e019526e7074560d57dd402781af", "lean_toolchain": "leanprover/lean4:v4.35.0", "llama_server_version": "0.5.0 (build 11146, commit 7fe450e19)", "p2b_artifact_commit": "6c40a4c3dea2380101dd1392ec6a344b1ff96dee", @@ -603,7 +603,7 @@ "public_holdout_commitment_sha256": "e7c9b9da52be68c4731e35961c31ec33decf440e4ccb00b2c9072e029054f032", "safetensors_checkpoint_fingerprint": "1ed4483f1d3ffbc3ffc7a26984be212e7cce455ac1e0651d1bcba1313ea2bd64", "spec_lineage_json_sha256": "344f5506e702dc8dc0ee7374d30af788e3d4c2c370fcdbc877435e133da26a76", - "tracked_evidence_merkle_sha256": "9477fe31b344bb3362988e99969434b5f57a81d9b5c71f09899de5aabc49cd5b", + "tracked_evidence_merkle_sha256": "841bb8a4234166c172ac43b5cfb7eb1c7b1c02599cb40e9b4add55fcd605184f", "vault_sha256sums_manifest_sha256": "b14cf1efeedf7a7c8c15ed57e587c3d18087f557e6ff1807068c4e9c9ef6f6ee", "zone_manifest_id": "sha256:4e00656cb47fce97f687ae638afba22c569b9680b3780eeb2e5ea71a62c55eca" }, @@ -1088,9 +1088,9 @@ "total": 15 }, "pytest_strict": { - "executed_total": 34, - "passed": 34, - "skipped": 0 + "executed_total": 53, + "passed": 53, + "skipped": 1 } }, "stack": { diff --git a/verification/graders/common.py b/verification/graders/common.py index 137990c..9bc06e5 100644 --- a/verification/graders/common.py +++ b/verification/graders/common.py @@ -15,6 +15,13 @@ import tempfile from typing import Any, Mapping, Sequence +from mcproof.lean_harness import ( + COMPILE_BLOCKING_LINT_RULES, + CandidateSandboxUnavailable, + lint_lean4_source, + run_confined_command, +) + getcontext().prec = 100 REPO_ROOT = Path(__file__).resolve().parents[2] @@ -274,6 +281,21 @@ def run_lean4_verification( "leanchecker_ran": False, } + # The soko auditor does not treat `attribute [init]` as a reason to stop. + # This lint does. The compile below is confined either way. + lint = lint_lean4_source(lean_source, require_theorem=False, check_vacuous=False) + for item in lint["findings"]: + if item.get("rule") in COMPILE_BLOCKING_LINT_RULES: + errors.append(f"lint:{item.get('rule')}:{item.get('message')}") + if errors: + return { + "ok": False, + "errors": errors, + "auditor": audit_rep, + "axioms": {}, + "leanchecker_ran": False, + } + tc_bin = find_lean_toolchain_bin() lean_exe = str(tc_bin / "lean") checker_exe = str(tc_bin / "leanchecker") @@ -300,14 +322,22 @@ def run_lean4_verification( olean_path = Path(td) / "CandidateMod.olean" mod_path.write_text(combined_source, encoding="utf-8") - comp = subprocess.run( - [lean_exe, "-o", str(olean_path), str(mod_path)], - cwd=td, - capture_output=True, - text=True, - env=env, - timeout=timeout_sec, - ) + try: + comp = run_confined_command( + [lean_exe, "-o", str(olean_path), str(mod_path)], + cwd=Path(td), + env=env, + timeout_sec=timeout_sec, + ) + except CandidateSandboxUnavailable as exc: + errors.append(f"sandbox_unavailable:{exc}") + return { + "ok": False, + "errors": errors, + "auditor": audit_rep, + "axioms": axioms_by_thm, + "leanchecker_ran": False, + } if comp.returncode != 0: msg = (comp.stdout + "\n" + comp.stderr).strip() errors.append(f"lean_compile_failed:{msg[:400]}") @@ -343,14 +373,22 @@ def run_lean4_verification( if not errors and run_leanchecker and Path(checker_exe).is_file(): env_chk = dict(env) env_chk["LEAN_PATH"] = f"{td}{os.pathsep}{libdir}" - chk_run = subprocess.run( - [checker_exe, "CandidateMod"], - cwd=td, - capture_output=True, - text=True, - env=env_chk, - timeout=timeout_sec, - ) + try: + chk_run = run_confined_command( + [checker_exe, "CandidateMod"], + cwd=Path(td), + env=env_chk, + timeout_sec=timeout_sec, + ) + except CandidateSandboxUnavailable as exc: + errors.append(f"sandbox_unavailable:{exc}") + return { + "ok": False, + "errors": errors, + "auditor": audit_rep, + "axioms": axioms_by_thm, + "leanchecker_ran": False, + } if chk_run.returncode != 0: msg = (chk_run.stdout + "\n" + chk_run.stderr).strip() errors.append(f"leanchecker_failed:{msg[:300]}") diff --git a/verification/oracles_verification_receipt.json b/verification/oracles_verification_receipt.json index 76ef1bd..84730ae 100644 --- a/verification/oracles_verification_receipt.json +++ b/verification/oracles_verification_receipt.json @@ -15,8 +15,8 @@ "float_true_division_cheat": { "actual_pass": false, "expected_pass": false, - "kill_fraction": "5/5", - "kill_rate_bps": 10000, + "kill_fraction": null, + "kill_rate_bps": null, "matched_expectation": true, "rules": [ "FLOAT_DIVISION_IN_EXACT_CHECKER", @@ -177,7 +177,7 @@ "native_decide_cheat": { "caught": true, "nonstandard_axioms": [ - "t_native._native.native_decide.ax_1_1" + "t_native._native.native_decide.ax_1" ], "rules": [ "UNVERIFIED_NATIVE_DECIDE", @@ -265,6 +265,7 @@ ], "challenge_sorry_whitelisted_module": "Mcproof.Spec", "findings": [], + "host_resource_exhausted": false, "nonstandard_axioms": [], "paranoid_kernels_run": [ "lean-paranoid", @@ -285,7 +286,28 @@ "Mcproof.Spec.ratLeCross_sound", "Mcproof.Spec.ratLeCross_refl_pos", "Mcproof.Spec.cleanSegments_safe", - "Mcproof.Spec.cleanSegments_singleton" + "Mcproof.Spec.cleanSegments_singleton", + "Mcproof.Spec.quotientHeuristic_step_consistent", + "Mcproof.Spec.quotientHeuristic_admissible", + "Mcproof.Spec.prefundedHeuristic_admissible_and_goal", + "Mcproof.Spec.prefundedHeuristic_step_consistent", + "Mcproof.Spec.prefundedVal_bounded_drop_witness", + "Mcproof.Spec.operatorCounting_dual_scaled_sound", + "Mcproof.Spec.operatorCounting_dual_div_admissible", + "Mcproof.Spec.dualFloorNat_sound_and_witness", + "Mcproof.Spec.cyclicSwitchTax_triangle", + "Mcproof.Spec.cyclicSwitchDist_metric_and_witness", + "Mcproof.Spec.conservation_harvest_potential_repair", + "Mcproof.Spec.vertexSplit_unit_flow_horizon_bound", + "Mcproof.Spec.zeroSelfLoop_deficit_composition", + "Mcproof.Spec.ceilDiv_step_consistent_and_admissible", + "Mcproof.Spec.rmsnorm_tangent_orthogonal", + "Mcproof.Spec.rmsnorm_orthogonal_stack_norm_inflation", + "Mcproof.Spec.multilayer_quadratic_shrinkage_reduction", + "Mcproof.Spec.swiglu_gate_metric_and_counterexample", + "Mcproof.Spec.swiglu_radial_null_orthogonality", + "Mcproof.Spec.scalar_projection_retention_illusion", + "Mcproof.Spec.rmsnorm_rational_shrinkage_sandwich" ], "toolchain": "leanprover/lean4:v4.35.0" }, @@ -317,7 +339,7 @@ "bytes": 106763, "control_bytes": [], "lineage_details": { - "corpus_tree_sha256": "5bdb7f30363b956e82947f668f98c48f99876e87fb5ed4f785b873e4f4f98305", + "corpus_tree_sha256": "93624aae09eea99c3d35ebfa2e9acde1f9ac9c675167461652486850f206b642", "deps_checkouts_verified": true, "deps_manifest_verified": true, "findings": [], @@ -333,15 +355,16 @@ "sha256": "49beff2a0185cfa4032170b614a543c0d67541494e858803d21cfb837111b230" }, "prose_gate": { - "check_dupes_out": "check-prose --dupes: clean (11 file(s))", + "check_dupes_out": "check-prose --dupes: clean (12 file(s))", "check_dupes_rc": 0, - "check_prose_out": "check-prose: clean (11 file(s), 2422 lines)", + "check_prose_out": "check-prose: clean (12 file(s), 2546 lines)", "check_prose_rc": 0, "files": [ "README.md", "compile/README.md", "datasets/README.md", "docs/CONTINUITY.md", + "docs/CROSS_POLLINATION.md", "lean/README.md", "lean/SPEC.md", "lean/tests/TEST-RESULTS.md", @@ -353,24 +376,24 @@ }, "pytest_strict": { "counts": { - "executed_total": 34, + "executed_total": 53, "failed": 0, - "passed": 34, - "skipped": 0 + "passed": 53, + "skipped": 1 }, "returncode": 0, "summary_tail": [ - "../../../Work/mcproof-lab/repo/tests/test_lean_harness.py ........... [ 73%]", - "../../../Work/mcproof-lab/repo/tests/test_lineage_prose_and_cli.py ..... [ 88%]", - ".... [100%]", + "tests/test_lean_harness.py ....s........ [ 70%]", + "tests/test_lineage_prose_and_cli.py ......... [ 87%]", + "tests/test_neon_audit_regressions.py ....... [100%]", "", - "======================== 34 passed in 137.54s (0:02:17) ========================" + "================== 53 passed, 1 skipped in 148.14s (0:02:28) ===================" ] }, "readme_top_lines": [ "# mcproof", "`mcproof` is the verification harness, Plane 1 rulebook, dataset suite, weight compiler, and serving benchmark for the Siliconcordat Formal Verification MDE. It checks Lean 4.35.0 proofs across six kernels, grades exact-math checkers against planted bugs, compiles `weft` adapters, and audits live `warp-modeld` mounts. Proof authors, verifiers, and researchers use it to test formal reasoning without weakening deterministic gates.", - "Blueprint and gates live in [siliconcordat/mde issue 4](https://github.com/siliconcordat/mde/issues/4) under the [Master Plan](https://github.com/siliconcordat/.github/blob/main/MASTER-PLAN.md). Companion serving changes live in [siliconcordat/exo](https://github.com/siliconcordat/exo/tree/feat/mde4-phase4-phase6-stack) and [siliconcordat/warp PR 5](https://github.com/siliconcordat/warp/pull/5)." + "Blueprint and gates live in [siliconcordat/mde issue 4](https://github.com/siliconcordat/mde/issues/4) under the [Master Plan](https://github.com/siliconcordat/.github/blob/main/MASTER-PLAN.md). Companion serving changes live in [siliconcordat/exo](https://github.com/siliconcordat/exo/tree/feat/mde4-phase4-phase6-stack) and [siliconcordat/warp](https://github.com/siliconcordat/warp/tree/feat/mde4-phase4-live-gate)." ], "siliconcordat_clean": { "exo": "",