From e257336f3ef33ca18c4c8cd9bc651aa0267d3866 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 17:10:09 +0000 Subject: [PATCH 1/2] Add a mutation-tested Layer A scan for every tracked Lean file The formal/ gate audits 9 of Math-'s 162 Lean files; the other 153 had no Layer A check, and a plain grep cannot tell `sorry` in code from the word in a comment (all 17 raw hits on main are prose). This scanner lexes comments and literals and reports forbidden names in code: sorry, admit, sorryAx, the axiom command, native_decide, ofReduceBool, trustCompiler, implemented_by, extern, unsafe, Float. Its tests plant each violation, check prose and literals stay silent, make malformed input a lex error, and kill 15 scanner mutants against an unmutated harness control. A new workflow runs them on any .lean change, which the full suite does not trigger on. First run: 0 findings in all 162 tracked Lean files. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017Mi3hxjaxV45x6zo6o1ee3 --- .github/workflows/layer-a-lean-scan.yml | 41 +++++ tests/test_lean_layer_a_scan.py | 193 ++++++++++++++++++++++++ tools/lean_layer_a_scan.py | 176 +++++++++++++++++++++ 3 files changed, 410 insertions(+) create mode 100644 .github/workflows/layer-a-lean-scan.yml create mode 100644 tests/test_lean_layer_a_scan.py create mode 100644 tools/lean_layer_a_scan.py diff --git a/.github/workflows/layer-a-lean-scan.yml b/.github/workflows/layer-a-lean-scan.yml new file mode 100644 index 000000000..9dc230901 --- /dev/null +++ b/.github/workflows/layer-a-lean-scan.yml @@ -0,0 +1,41 @@ +name: Layer A Lean scan + +# CBZ-0 gates G0 and G1: mutation-test the Layer A scanner, then scan every tracked +# Lean file for placeholders, axiom injection, trust escapes and floating point in code. +# Static only: it does not replace a kernel build or #print axioms. +'on': + pull_request: + paths: + - '**/*.lean' + - '.github/workflows/layer-a-lean-scan.yml' + - 'tools/lean_layer_a_scan.py' + - 'tests/test_lean_layer_a_scan.py' + push: + branches: [main] + paths: + - '**/*.lean' + - 'tools/lean_layer_a_scan.py' + - 'tests/test_lean_layer_a_scan.py' + workflow_dispatch: + +permissions: + contents: read + +jobs: + scan: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 + with: + python-version: '3.11.16' + - name: G0 scanner self-test and G1 repository scan, both modes + env: + PYTHONDONTWRITEBYTECODE: '1' + run: | + python -B -S tests/test_lean_layer_a_scan.py + python -B -O -S tests/test_lean_layer_a_scan.py + python -B -S tools/lean_layer_a_scan.py --root . diff --git a/tests/test_lean_layer_a_scan.py b/tests/test_lean_layer_a_scan.py new file mode 100644 index 000000000..afa1bb42c --- /dev/null +++ b/tests/test_lean_layer_a_scan.py @@ -0,0 +1,193 @@ +"""Self-tests for tools/lean_layer_a_scan.py (CBZ-0 gate G0). + +The scanner certifies Layer A, so it must itself be shown to catch what it claims: +planted violations in every code position are found, the same text in comments and +literals is not, malformed input is a lex error rather than a silent pass, and every +mutant of the scanner's own source is killed by this suite. +""" +from __future__ import annotations + +import pathlib +import subprocess +import sys +import tempfile +import unittest + +ROOT = pathlib.Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tools')) +import lean_layer_a_scan as scan # noqa: E402 + +SCANNER = ROOT / 'tools' / 'lean_layer_a_scan.py' + + +def hits(text): + return [(f['line'], f['token']) for f in scan.findings_for_text(text)] + + +class CodePositionTests(unittest.TestCase): + def test_every_forbidden_name_is_found_in_code(self): + for token, invariant in scan.FORBIDDEN.items(): + with self.subTest(token=token): + found = scan.findings_for_text('theorem t : True := by\n ' + token + '\n') + self.assertEqual([(f['line'], f['token'], f['invariant']) for f in found], + [(2, token, invariant)]) + + def test_forms_of_placeholders_and_escapes(self): + cases = { + 'theorem a : 1 = 1 := sorry': [(1, 'sorry')], + 'example : P := by admit': [(1, 'admit')], + 'axiom bad : False': [(1, 'axiom')], + 'example : 2 + 2 = 4 := by native_decide': [(1, 'native_decide')], + 'def x := Lean.ofReduceBool': [(1, 'ofReduceBool')], + 'def y : Float := 1.5': [(1, 'Float')], + 'def z := Float.ofScientific 1 true 1': [(1, 'Float')], + '@[implemented_by foo] def w := 0': [(1, 'implemented_by')], + '@[extern "c"] opaque e : Nat': [(1, 'extern')], + 'unsafe def u := 0': [(1, 'unsafe')], + 'def q := «sorry»': [(1, 'sorry')], + 'def r := Foo.«sorryAx»': [(1, 'sorryAx')], + 'theorem s : P := (sorry)': [(1, 'sorry')], + } + for text, expected in cases.items(): + with self.subTest(text=text): + self.assertEqual(hits(text), expected) + + def test_line_numbers_follow_comments_and_strings(self): + text = '/- one\ntwo -/\ndef s := "a\nb"\n-- c\ntheorem t : P := sorry\n' + self.assertEqual(hits(text), [(6, 'sorry')]) + + def test_code_after_a_comment_on_the_same_line(self): + self.assertEqual(hits('/- c -/ sorry'), [(1, 'sorry')]) + self.assertEqual(hits('def a := "x" ++ sorry'), [(1, 'sorry')]) + + +class NonCodeTests(unittest.TestCase): + def test_prose_and_literals_are_ignored(self): + for text in ( + '-- these densities admit versions; no sorry here', + '/-- docstring: uses no `sorry`, `native_decide` or Float. -/', + '/-! module doc: axiom audit is separate -/', + '/- outer /- nested sorry -/ still comment admit -/', + 'def s := "sorry admit axiom Float"', + 'def t := "escaped \\" quote sorry"', + "def c := 'a'", + "def d := '\\n'", + 'def e := "contains -- and /- not comments"', + ): + with self.subTest(text=text): + self.assertEqual(hits(text), []) + + def test_identifiers_that_merely_contain_a_name_are_not_hits(self): + for text in ('def sorryCount := 0', 'theorem admits_bound : P := h', + 'def axiomatic := 0', 'def Floating := 0', "def x' := sorry'"): + with self.subTest(text=text): + self.assertEqual(hits(text), []) + + def test_text_after_a_comment_close_is_code_again(self): + self.assertEqual(hits('/- a /- b -/ c -/ admit'), [(1, 'admit')]) + + +class MalformedInputTests(unittest.TestCase): + def test_unterminated_regions_raise(self): + for text in ('/- never closed sorry', '/- /- one level closed -/ sorry', + 'def s := "never closed', 'def q := «never closed'): + with self.subTest(text=text): + with self.assertRaises(scan.LexError): + scan.findings_for_text(text) + + def test_cli_reports_lex_errors_with_exit_two_and_findings_with_exit_one(self): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + (root / 'ok.lean').write_text('theorem t : True := trivial\n') + (root / 'bad.lean').write_text('theorem t : P := sorry\n') + (root / 'broken.lean').write_text('/- open\n') + run = lambda *p: subprocess.run([sys.executable, '-I', str(SCANNER), '--root', td, *p], + capture_output=True, text=True) + self.assertEqual(run('ok.lean').returncode, 0) + bad = run('ok.lean', 'bad.lean') + self.assertEqual(bad.returncode, 1) + self.assertIn('bad.lean:1: A1 sorry', bad.stdout) + broken = run('ok.lean', 'bad.lean', 'broken.lean') + self.assertEqual(broken.returncode, 2) + self.assertIn('broken.lean: LEX ERROR unterminated block comment', broken.stdout) + + def test_default_scope_is_every_tracked_lean_file_outside_lake(self): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + for cmd in (['init', '-q'],): + subprocess.run(['git', *cmd], cwd=td, check=True) + (root / 'a').mkdir() + (root / 'a' / 'x.lean').write_text('def x := sorry\n') + (root / '.lake').mkdir() + (root / '.lake' / 'y.lean').write_text('def y := sorry\n') + (root / 'untracked.lean').write_text('def z := sorry\n') + subprocess.run(['git', 'add', 'a/x.lean', '-f', '.lake/y.lean'], cwd=td, check=True) + self.assertEqual(scan.tracked_lean_files(root), ['a/x.lean']) + + +class RepositoryScanTests(unittest.TestCase): + """G1 on this tree: no tracked Lean file has a forbidden name in code.""" + + def test_tracked_lean_files_have_no_layer_a_findings(self): + files = scan.tracked_lean_files(ROOT) + self.assertGreater(len(files), 0, 'no tracked Lean files found: the scan would be vacuous') + report = scan.scan(ROOT, files) + self.assertEqual(report['lex_errors'], []) + self.assertEqual(report['findings'], []) + + +MUTANTS = [ + # (description, old, new): each must make this suite fail. + ('drop sorry', "'sorry': 'A1', ", ''), + ('drop admit', "'admit': 'A1', ", ''), + ('drop axiom', "'axiom': 'A2',", ''), + ('drop native_decide', "'native_decide': 'A2', ", ''), + ('drop Float', "'Float': 'A3', ", ''), + ('whole name only, no components', 'hits = [p for p in parts if p in FORBIDDEN]', + 'hits = [name] if name in FORBIDDEN else []'), + ('block comments not nested', 'depth, i = depth + 1, i + 2', 'i = i + 2'), + ('unterminated comment passes', "raise LexError('unterminated block comment", "return\n raise LexError('x"), + ('string escapes ignored', "if text[i] == '\\\\':", "if False:"), + ('line comments swallow the file', "i = n if j < 0 else j", "i = n"), + ('lex errors exit 0', "return 2 if report['lex_errors'] else", "return 0 if report['lex_errors'] else"), + ('findings exit 0', "else 1 if report['findings'] else 0", "else 0"), + ('lake not excluded', "if p and '/.lake/' not in '/' + p", "if p"), + ('guillemets dropped', "parts.append(text[i + 1:j])", "parts.append('')"), + ('line count frozen', "line += 1\n i += 1", "i += 1"), +] + + +class ScannerMutationTests(unittest.TestCase): + """Run this suite (minus this class) against mutated scanners; every mutant must fail.""" + + def run_suite(self, scanner_source): + here = pathlib.Path(__file__).read_text() + with tempfile.TemporaryDirectory() as td: + tools = pathlib.Path(td) / 'tools' + tests = pathlib.Path(td) / 'tests' + tools.mkdir() + tests.mkdir() + (tools / 'lean_layer_a_scan.py').write_text(scanner_source) + (tests / 'test_lean_layer_a_scan.py').write_text(here) + return subprocess.run( + [sys.executable, '-B', '-m', 'unittest', '-q', 'test_lean_layer_a_scan.CodePositionTests', + 'test_lean_layer_a_scan.NonCodeTests', 'test_lean_layer_a_scan.MalformedInputTests'], + cwd=tests, capture_output=True, text=True, timeout=120) + + def test_harness_control_unmutated_scanner_passes(self): + # Without this, an import or harness failure would make every mutant look killed. + run = self.run_suite(SCANNER.read_text()) + self.assertEqual(run.returncode, 0, run.stderr[-2000:]) + + def test_every_mutant_is_killed(self): + source = SCANNER.read_text() + for desc, old, new in MUTANTS: + with self.subTest(mutant=desc): + self.assertEqual(source.count(old), 1, 'mutation anchor missing or ambiguous: ' + desc) + run = self.run_suite(source.replace(old, new)) + self.assertNotEqual(run.returncode, 0, 'mutant survived: ' + desc) + self.assertIn('FAILED', run.stderr, 'mutant died without a test failure: ' + desc) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lean_layer_a_scan.py b/tools/lean_layer_a_scan.py new file mode 100644 index 000000000..3c9165be7 --- /dev/null +++ b/tools/lean_layer_a_scan.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Layer A source scan for every tracked Lean file (CBZ-0 gate G1). + +Lexes Lean 4 source well enough to tell code from comments and literals, then reports +every code occurrence of a forbidden name: + + A1 placeholders: sorry, admit, sorryAx + A2 axiom injection: the `axiom` command (and `axioms` is not a keyword, so only `axiom`) + A1/A2 trust escapes: native_decide, ofReduceBool, trustCompiler, implemented_by, + extern, unsafe + A3 floating point: Float, Float32 + +A name matches when the whole identifier, or any dotted component of it, is forbidden +(`Lean.ofReduceBool`, `Float.ofScientific`), including «guillemet»-quoted identifiers. +Text inside line comments (`--`), nested block and doc comments (`/- -/`, `/-- -/`, +`/-! -/`), string literals and character literals is ignored, so prose such as +"densities admit versions" is not a hit. + +This is a static scan. It does not replace a kernel build or `#print axioms`; it closes +the gap for Lean files that no gated package compiles. Standard library only. + +Usage: lean_layer_a_scan.py [--root DIR] [--json] [PATH ...] +With no PATH it scans every `*.lean` file tracked by git under the root, skipping `.lake/`. +Exit status: 0 no findings, 1 findings, 2 a file could not be lexed (unterminated +comment or string): that file is reported, never silently skipped. +""" +from __future__ import annotations + +import argparse +import json +import pathlib +import subprocess +import sys + +FORBIDDEN = { + 'sorry': 'A1', 'admit': 'A1', 'sorryAx': 'A1', + 'axiom': 'A2', + 'native_decide': 'A2', 'ofReduceBool': 'A2', 'trustCompiler': 'A2', + 'implemented_by': 'A2', 'extern': 'A2', 'unsafe': 'A2', + 'Float': 'A3', 'Float32': 'A3', +} + + +class LexError(ValueError): + pass + + +def _ident_start(c: str) -> bool: + return c.isalpha() or c == '_' + + +def _ident_char(c: str) -> bool: + return c.isalnum() or c in "_'!?" + + +def code_identifiers(text: str): + """Yield (line, name) for each identifier outside comments and literals. + + A dotted name such as `Lean.ofReduceBool` is yielded as one name. Raises LexError on + an unterminated block comment, string or guillemet identifier. + """ + i, n, line = 0, len(text), 1 + while i < n: + c = text[i] + if c == '\n': + line += 1 + i += 1 + elif text.startswith('--', i): + j = text.find('\n', i) + i = n if j < 0 else j + elif text.startswith('/-', i): + depth, start, i = 1, line, i + 2 + while depth: + if i >= n: + raise LexError('unterminated block comment opened on line %d' % start) + if text.startswith('/-', i): + depth, i = depth + 1, i + 2 + elif text.startswith('-/', i): + depth, i = depth - 1, i + 2 + else: + line += text[i] == '\n' + i += 1 + elif c == '"': + start, i = line, i + 1 + while True: + if i >= n: + raise LexError('unterminated string opened on line %d' % start) + if text[i] == '\\': + line += text[i + 1:i + 2] == '\n' + i += 2 + elif text[i] == '"': + i += 1 + break + else: + line += text[i] == '\n' + i += 1 + elif c == "'" and i + 2 < n and (text[i + 2] == "'" or (text[i + 1] == '\\' and "'" in text[i + 3:i + 12])): + # character literal 'a' or '\n' / '\x41' / '\u{..}' + i = text.index("'", i + 2 if text[i + 1] != '\\' else i + 3) + 1 + elif c == '«' or _ident_start(c): + parts, start = [], line + while True: + if i < n and text[i] == '«': + j = text.find('»', i) + if j < 0: + raise LexError('unterminated guillemet identifier on line %d' % start) + parts.append(text[i + 1:j]) + i = j + 1 + elif i < n and _ident_start(text[i]): + j = i + while j < n and _ident_char(text[j]): + j += 1 + parts.append(text[i:j]) + i = j + else: + break + if i + 1 < n and text[i] == '.' and (text[i + 1] == '«' or _ident_start(text[i + 1])): + i += 1 + else: + break + yield start, '.'.join(parts), parts + else: + i += 1 + + +def findings_for_text(text: str): + found = [] + for line, name, parts in code_identifiers(text): + hits = [p for p in parts if p in FORBIDDEN] + if hits: + found.append({'line': line, 'name': name, 'token': hits[0], 'invariant': FORBIDDEN[hits[0]]}) + return found + + +def tracked_lean_files(root: pathlib.Path): + out = subprocess.run(['git', '-C', str(root), 'ls-files', '-z', '--', '*.lean'], + check=True, capture_output=True).stdout + return sorted(p for p in out.decode().split('\0') if p and '/.lake/' not in '/' + p) + + +def scan(root: pathlib.Path, paths): + report = {'files': 0, 'findings': [], 'lex_errors': []} + for rel in paths: + report['files'] += 1 + try: + text = (root / rel).read_text(encoding='utf-8') + for f in findings_for_text(text): + report['findings'].append(dict(f, path=rel)) + except (LexError, UnicodeDecodeError) as exc: + report['lex_errors'].append({'path': rel, 'error': str(exc)}) + return report + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split('\n')[0]) + ap.add_argument('--root', default='.') + ap.add_argument('--json', action='store_true') + ap.add_argument('paths', nargs='*') + args = ap.parse_args(argv) + root = pathlib.Path(args.root).resolve() + paths = args.paths or tracked_lean_files(root) + report = scan(root, paths) + if args.json: + print(json.dumps(report, indent=1, sort_keys=True)) + else: + for f in report['findings']: + print('%s:%d: %s %s (%s)' % (f['path'], f['line'], f['invariant'], f['name'], f['token'])) + for e in report['lex_errors']: + print('%s: LEX ERROR %s' % (e['path'], e['error'])) + print('scanned %d Lean files: %d findings, %d lex errors' + % (report['files'], len(report['findings']), len(report['lex_errors']))) + return 2 if report['lex_errors'] else 1 if report['findings'] else 0 + + +if __name__ == '__main__': + sys.exit(main()) From 6d402474a4a9582a151e9f87ab5f69570a2bc58b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 17:20:52 +0000 Subject: [PATCH 2/2] Fix interpolation-hole and raw-string lexing (Codex 4415546474, 4415546483) The scanner treated every "..." as an opaque literal, so code in the {...} holes of s!/m!/f! interpolated strings was never scanned (P1), and raw strings r"..." / r#"..."# were lexed with escapes, so a backslash or an embedded quote could hide or invent code (P2). Holes are now lexed recursively as code with their line offsets; raw strings close only at '"' plus the same number of hashes and take no escapes. Unclosed raw strings and holes are lex errors (exit 2). Tests: interpolation and raw-string positives and negatives, malformed inputs, a direct test of the hole helper, and five new scanner mutants (20 in total), each killed with the unmutated harness as control. Both interpreter modes OK; the tracked tree still scans 162 Lean files with 0 findings and 0 lex errors. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017Mi3hxjaxV45x6zo6o1ee3 --- tests/test_lean_layer_a_scan.py | 34 +++++++++++++++++++++- tools/lean_layer_a_scan.py | 50 +++++++++++++++++++++++++++++++-- 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/tests/test_lean_layer_a_scan.py b/tests/test_lean_layer_a_scan.py index afa1bb42c..b51e861c1 100644 --- a/tests/test_lean_layer_a_scan.py +++ b/tests/test_lean_layer_a_scan.py @@ -56,6 +56,24 @@ def test_line_numbers_follow_comments_and_strings(self): text = '/- one\ntwo -/\ndef s := "a\nb"\n-- c\ntheorem t : P := sorry\n' self.assertEqual(hits(text), [(6, 'sorry')]) + def test_interpolation_holes_are_code(self): + # Codex review 4415546474: the holes of s!/m!/f! strings are expressions. + self.assertEqual(hits('def s := s!"{(sorry : Nat)}"'), [(1, 'sorry')]) + self.assertEqual(hits('def s := m!"x {f admit} y"'), [(1, 'admit')]) + self.assertEqual(hits('def s := s!"a\n{\n (sorry : Nat)}"'), [(3, 'sorry')]) + self.assertEqual(hits('def s := s!"{g {h := sorry}}"'), [(1, 'sorry')]) + self.assertEqual(hits('def s := s!"literal sorry {x} admit"'), []) + self.assertEqual(hits('def s := s!"escaped \\{sorry}"'), []) + self.assertEqual(hits('def s := "{sorry}"'), []) + + def test_raw_strings(self): + # Codex review 4415546483: raw strings have no escapes and may use # delimiters. + self.assertEqual(hits('def s := r#"a "sorry" b"#'), []) + self.assertEqual(hits('def s := r"\\"\ntheorem t : False := sorry\n-- "'), [(2, 'sorry')]) + self.assertEqual(hits('def s := r##"x "# sorry"##\ndef u := admit'), [(2, 'admit')]) + self.assertEqual(hits('def r := sorry'), [(1, 'sorry')]) + self.assertEqual(hits('def x := bar"y"'), []) + def test_code_after_a_comment_on_the_same_line(self): self.assertEqual(hits('/- c -/ sorry'), [(1, 'sorry')]) self.assertEqual(hits('def a := "x" ++ sorry'), [(1, 'sorry')]) @@ -88,9 +106,18 @@ def test_text_after_a_comment_close_is_code_again(self): class MalformedInputTests(unittest.TestCase): + def test_unclosed_interpolation_hole_raises(self): + # Through the full lexer an unclosed hole also leaves its string unterminated, so + # the helper's own refusal is pinned directly. + for text in ('{abc', '{a {b} c', '{"}" x'): + with self.subTest(text=text), self.assertRaises(scan.LexError): + scan._hole_end(text, 0) + self.assertEqual(scan._hole_end('{a {b} "}" c} d', 0), 12) + def test_unterminated_regions_raise(self): for text in ('/- never closed sorry', '/- /- one level closed -/ sorry', - 'def s := "never closed', 'def q := «never closed'): + 'def s := "never closed', 'def q := «never closed', + 'def r := r#"never closed"', 'def h := s!"{unclosed"'): with self.subTest(text=text): with self.assertRaises(scan.LexError): scan.findings_for_text(text) @@ -154,6 +181,11 @@ def test_tracked_lean_files_have_no_layer_a_findings(self): ('lake not excluded', "if p and '/.lake/' not in '/' + p", "if p"), ('guillemets dropped', "parts.append(text[i + 1:j])", "parts.append('')"), ('line count frozen', "line += 1\n i += 1", "i += 1"), + ('interpolation holes skipped', "elif interpolated and text[i] == '{':", "elif False:"), + ('interpolation never detected', "interpolated = i >= 2 and", "interpolated = False and"), + ('raw strings not recognised', "elif c == 'r' and RAW_OPEN.match(text, i)", "elif False and RAW_OPEN.match(text, i)"), + ('raw string hashes ignored', "start, close = line, '\"' + hashes", "start, close = line, '\"'"), + ('unclosed hole passes', "raise LexError('unterminated interpolation hole')", "return n"), ] diff --git a/tools/lean_layer_a_scan.py b/tools/lean_layer_a_scan.py index 3c9165be7..f5738c184 100644 --- a/tools/lean_layer_a_scan.py +++ b/tools/lean_layer_a_scan.py @@ -13,8 +13,10 @@ A name matches when the whole identifier, or any dotted component of it, is forbidden (`Lean.ofReduceBool`, `Float.ofScientific`), including «guillemet»-quoted identifiers. Text inside line comments (`--`), nested block and doc comments (`/- -/`, `/-- -/`, -`/-! -/`), string literals and character literals is ignored, so prose such as -"densities admit versions" is not a hit. +`/-! -/`), string literals, raw strings (`r"..."`, `r#"..."#`, which have no escapes) and +character literals is ignored, so prose such as "densities admit versions" is not a hit. +The `{...}` holes of interpolated strings (`s!"..."`, `m!"..."`, `f!"..."`) are code and +are scanned. This is a static scan. It does not replace a kernel build or `#print axioms`; it closes the gap for Lean files that no gated package compiles. Standard library only. @@ -29,6 +31,7 @@ import argparse import json import pathlib +import re import subprocess import sys @@ -53,6 +56,32 @@ def _ident_char(c: str) -> bool: return c.isalnum() or c in "_'!?" +RAW_OPEN = re.compile(r'r(#*)"') + + +def _hole_end(text: str, i: int) -> int: + """Index of the '}' closing the interpolation hole that opens at text[i] == '{'. + + Nested braces and string literals inside the hole are skipped. An unclosed hole is a + LexError, so a malformed interpolation can never hide code. + """ + depth, j, n = 0, i, len(text) + while j < n: + ch = text[j] + if ch == '{': + depth += 1 + elif ch == '}': + depth -= 1 + if depth == 0: + return j + elif ch == '"': + j += 1 + while j < n and text[j] != '"': + j += 2 if text[j] == '\\' else 1 + j += 1 + raise LexError('unterminated interpolation hole') + + def code_identifiers(text: str): """Yield (line, name) for each identifier outside comments and literals. @@ -80,7 +109,18 @@ def code_identifiers(text: str): else: line += text[i] == '\n' i += 1 + elif c == 'r' and RAW_OPEN.match(text, i) and (i == 0 or not (_ident_char(text[i - 1]) or text[i - 1] == '.')): + # raw string r"..." / r#"..."#: no escapes; closes at '"' followed by the same hashes + hashes = RAW_OPEN.match(text, i).group(1) + start, close = line, '"' + hashes + j = text.find(close, i + 2 + len(hashes)) + if j < 0: + raise LexError('unterminated raw string opened on line %d' % start) + line += text.count('\n', i, j) + i = j + len(close) elif c == '"': + # s!"..{e}.." / m!"..." / f!"...": the braced holes are code and are scanned + interpolated = i >= 2 and text[i - 1] == '!' and _ident_char(text[i - 2]) start, i = line, i + 1 while True: if i >= n: @@ -91,6 +131,12 @@ def code_identifiers(text: str): elif text[i] == '"': i += 1 break + elif interpolated and text[i] == '{': + j = _hole_end(text, i) + for hole_line, name, parts in code_identifiers(text[i + 1:j]): + yield line + hole_line - 1, name, parts + line += text.count('\n', i, j) + i = j + 1 else: line += text[i] == '\n' i += 1