From eed741581eed4a0ff499b24778451575fb40557c Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Wed, 7 Oct 2026 01:04:41 -0700 Subject: [PATCH 1/3] Preserve regex literals after contextual for-of separators Track bounded iteration-header context without turning ordinary of identifiers into regex prefixes. Preserve existing ambiguity refusals and add parent-red regression controls. BQ42 author candidate; scientific effect NONE. --- tests/test_site_asset_release.py | 149 +++++++++++++++++++++++++++++++ tools/site_asset_release.py | 41 +++++++-- 2 files changed, 184 insertions(+), 6 deletions(-) diff --git a/tests/test_site_asset_release.py b/tests/test_site_asset_release.py index 57b1a20c9..707056354 100644 --- a/tests/test_site_asset_release.py +++ b/tests/test_site_asset_release.py @@ -364,6 +364,155 @@ def test_unsupported_contexts_fail_before_any_writes(self): self.assertNotEqual(result.returncode, 0) self.assertEqual(before, {p.name:p.read_bytes() for p in site.iterdir()}) + def test_iteration_header_regex_has_no_module_spans(self): + pattern = '/import("ghost.mjs?site-release=keep")/' + examples = [ + f'for (const x of {pattern}) {{}}', + f'for (let x of {pattern}) {{}}', + f'for (var x of {pattern}) {{}}', + f'for (x of {pattern}) {{}}', + f'for (of of {pattern}) {{}}', + f'for (const of of {pattern}) {{}}', + f'for (const [x, {{y}}] of {pattern}) {{}}', + f'for ({{x}} of {pattern}) {{}}', + f'for (obj.of of {pattern}) {{}}', + f'for (obj.in of {pattern}) {{}}', + f'for (obj.for of {pattern}) {{}}', + f'for (obj.return of {pattern}) {{}}', + f'for (obj.let of {pattern}) {{}}', + f'for (obj[key] of {pattern}) {{}}', + f'for ((x) of {pattern}) {{}}', + f'for /*a*/ (const x /*b*/ of //c\n {pattern}) {{}}', + f'for await (const x of {pattern}) {{}}', + f'for /*a*/ await /*b*/ (const x of /*c*/ {pattern}) {{}}', + f'for (const x of xs) for (const y of {pattern}) {{}}', + f'for (let x = (() => {{ for (const y of {pattern}) {{}} }})();; ) {{}}', + f'for (const x of ({pattern})) {{}}', + 'for (const x of /[(){}]import("ghost.mjs")/g) {}', + ] + for source in examples: + with self.subTest(source=source): + self.assertEqual(asset_release.source_spans(source, '.mjs'), []) + self.assertEqual(asset_release.transform( + source, '.mjs', asset_release.without_release), source) + + def test_iteration_regex_cli_preserves_tracked_and_missing_pattern_targets(self): + for header in ('for', 'for await'): + for tracked in (False, True): + with self.subTest(header=header, tracked=tracked), TemporaryDirectory() as tmp: + root = Path(tmp) + subprocess.run(['git', 'init', '-q', str(root)], check=True, + capture_output=True) + site = root/'site'; site.mkdir() + source = f'{header} (const x of /import("core.mjs")/) {{}}\n' + (site/'app.mjs').write_text(source) + if tracked: (site/'core.mjs').write_text('export const x = 1;\n') + self.stage(root) + before = self.snapshot(root) + token = None + for args in [('--check',), (), ('--check',), ()]: + result = self.run_tool(site, *args) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(result.stderr, '') + self.assertRegex(result.stdout, r'^[0-9a-f]{64}\n$') + if token is None: token = result.stdout + self.assertEqual(result.stdout, token) + self.assertEqual(self.snapshot(root), before) + + def test_identifier_of_division_still_discovers_real_imports(self): + live = 'import("./core.mjs")' + examples = [ + f'const of = 2; of / {live} / 2;', + f'obj.of / {live} / 2; obj?.of / {live} / 2;', + f'for (of / {live} / 2; ; ) {{}}', + f'for (let x = of / {live} / 2; ; ) {{}}', + f'for (let of = 2; of / {live} / 2; of / {live} / 2) {{}}', + f'for (x in of / {live} / 2) {{}}', + f'for (x of of / {live} / 2) {{}}', + f'for await (x of of / {live} / 2) {{}}', + f'for (x of obj.of / {live} / 2) {{}}', + f'for (x of obj.in / {live} / 2) {{}}', + f'for (x of obj?.of / {live} / 2) {{}}', + f'for (x of obj[of / {live} / 2]) {{}}', + f'for (let x = {{of: of / {live} / 2}}; ; ) {{}}', + f'for (let x = (of / {live} / 2); ; ) {{}}', + f'for (let x = [of / {live} / 2]; ; ) {{}}', + f'for (let x = of; x; x += of / {live} / 2) {{}}', + f'for (let x = typeof of / {live} / 2; ; ) {{}}', + f'obj.for(of / {live} / 2); obj?.for(of / {live} / 2);', + f'obj.for(ok) / scale / {live}; obj?.for(ok) / scale / {live};', + f'for (x of xs) {{}} of / {live} / 2;', + f'for (obj[of / {live} / 2] of values) {{}}', + f'for (const [x = of / {live} / 2] of values) {{}}', + ] + for source in examples: + with self.subTest(source=source): + spans = asset_release.source_spans(source, '.mjs') + self.assertEqual([source[a:b] for a, b in spans], + ['./core.mjs'] * source.count(live)) + self.assertEqual(asset_release.transform( + source, '.mjs', lambda value: value + '?site-release=new'), + source.replace('./core.mjs', './core.mjs?site-release=new')) + + def test_iteration_regex_and_real_imports_have_distinct_cli_behavior(self): + pattern = '/import("ghost.mjs?site-release=keep")/' + source = (f'for (const x of {pattern}) {{ import("./core.mjs"); }}\n' + 'for (x of of / import("./core.mjs") / 2) {}\n') + with TemporaryDirectory() as tmp: + root = Path(tmp); site = self.fixture(root) + (site/'app.mjs').write_text(source) + first = self.run_tool(site) + self.assertEqual(first.returncode, 0, first.stderr) + expected = source.replace('./core.mjs', './core.mjs?site-release=' + first.stdout.strip()) + self.assertEqual((site/'app.mjs').read_text(), expected) + before = self.snapshot(root) + for args in [('--check',), ()]: + result = self.run_tool(site, *args) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, first.stdout) + self.assertEqual(self.snapshot(root), before) + + def test_missing_iteration_division_import_refuses_before_writes(self): + source = ('for (const x of /import("core.mjs")/) {}\n' + 'for (x of of / import("./missing.mjs") / 2) {}\n') + with TemporaryDirectory() as tmp: + root = Path(tmp); site = self.fixture(root) + (site/'app.mjs').write_text(source) + before = self.snapshot(root) + for args in [('--check',), ()]: + result = self.run_tool(site, *args) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertEqual(result.stdout, '') + self.assertIn('missing/outside local asset ./missing.mjs', result.stderr) + self.assertEqual(self.snapshot(root), before) + + def test_iteration_regex_refusals_preserve_all_files_and_index(self): + examples = [ + ('for (const x of /import("core.mjs")\n) {}', + 'Unsupported or unterminated regex'), + ('for await (const x of /import("core.mjs")\n) {}', + 'Unsupported or unterminated regex'), + ('for (const \\u0078 of /import("core.mjs")/) {}', + 'Unsupported JavaScript identifier syntax'), + ('for (const x of xs) {} /import("core.mjs")/.test(s);', + 'Ambiguous regex/division containing module syntax'), + ('for (obj[of / import("./core.mjs") / 2] of /import("ghost.mjs")/) {}', + 'Ambiguous regex/division containing module syntax'), + ('for (const [x = of / import("./core.mjs") / 2] of /import("ghost.mjs")/) {}', + 'Ambiguous regex/division containing module syntax'), + ] + for source, message in examples: + with self.subTest(source=source), TemporaryDirectory() as tmp: + root = Path(tmp); site = self.fixture(root) + (site/'app.mjs').write_text('import "./core.mjs";\n' + source) + before = self.snapshot(root) + for args in [('--check',), ()]: + result = self.run_tool(site, *args) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertEqual(result.stdout, '') + self.assertIn(message, result.stderr) + self.assertEqual(self.snapshot(root), before) + def test_untracked_dependency_and_non_git_site_are_refused(self): with TemporaryDirectory() as tmp: root = Path(tmp); site = self.fixture(root) diff --git a/tools/site_asset_release.py b/tools/site_asset_release.py index e573d7118..889ef8f54 100644 --- a/tools/site_asset_release.py +++ b/tools/site_asset_release.py @@ -230,6 +230,7 @@ def _js_tokens(text, start=0, template_expression=False): tokens = [] pos = start braces = 0 + brackets = 0 parens = [] previous = None while pos < len(text): @@ -269,7 +270,7 @@ def _js_tokens(text, start=0, template_expression=False): property_word = (previous is not None and previous[0] == 'word' and len(tokens) >= 2 and tokens[-2][1] in ('.', '?.')) - prefix = (previous is None or previous[0] == 'control-close' + prefix = (previous is None or previous[0] in ('control-close', 'for-of') or (previous[1] in _REGEX_PREFIX and not property_word)) ambiguous = previous is not None and previous[1] in (')', '}') if prefix: @@ -302,19 +303,47 @@ def _js_tokens(text, start=0, template_expression=False): pos = match.end() token = ('punct', match[0], begin, pos) + # `of` is a separator only in the binding part of this for header. + # In particular, an identifier named `of` can precede real division + # in a classic for clause, a nested expression, or the iterable RHS. + if (parens and parens[-1]['for-binding'] + and (braces, brackets) == parens[-1]['depth']): + property_name = previous is not None and previous[1] in ('.', '?.') + previous_property = len(tokens) >= 2 and tokens[-2][1] in ('.', '?.') + target_end = previous is not None and ( + previous[1] in (')', ']', '}') + or (previous[0] == 'word' and ( + previous_property or (previous[1] not in _REGEX_PREFIX + and previous[1] not in ('const', 'let', 'var', 'using'))))) + if token[1] == ';' or (token[1] == 'in' and not property_name): + parens[-1]['for-binding'] = False + elif token[1] == 'of' and not property_name and target_end: + token = ('for-of', 'of', begin, pos) + parens[-1]['for-binding'] = False + if token[1] == '(': - parens.append(previous is not None - and previous[0] == 'word' - and previous[1] in _CONTROL - and (len(tokens) < 2 or tokens[-2][1] not in ('.', '?.'))) + head = len(tokens) - 1 + if (head >= 1 and tokens[head][1] == 'await' + and tokens[head - 1][1] == 'for'): + head -= 1 + control = (head >= 0 and tokens[head][0] == 'word' + and tokens[head][1] in _CONTROL + and (head == 0 or tokens[head - 1][1] not in ('.', '?.'))) + parens.append({'control': control, + 'for-binding': control and tokens[head][1] == 'for', + 'depth': (braces, brackets)}) elif token[1] == ')': - control = parens.pop() if parens else False + control = parens.pop()['control'] if parens else False if control: token = ('control-close', ')', begin, pos) elif token[1] == '{': braces += 1 elif token[1] == '}': braces -= 1 + elif token[1] == '[': + brackets += 1 + elif token[1] == ']': + brackets -= 1 tokens.append(token) previous = token if template_expression: From 2fbc48917047e7c4b354f10d8e51b9b19eaaa27b Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Wed, 7 Oct 2026 01:40:55 -0700 Subject: [PATCH 2/3] Tokenize private names separately from JavaScript keywords Preserve private-member iteration regexes and distinguish private calls from genuine imports and division. Refuse unsupported hash syntax before writes. BQ42 amendment of EXEC-1; author candidate, scientific effect NONE. --- tests/test_site_asset_release.py | 146 +++++++++++++++++++++++++++++++ tools/site_asset_release.py | 10 ++- 2 files changed, 155 insertions(+), 1 deletion(-) diff --git a/tests/test_site_asset_release.py b/tests/test_site_asset_release.py index 707056354..4e700f352 100644 --- a/tests/test_site_asset_release.py +++ b/tests/test_site_asset_release.py @@ -24,6 +24,17 @@ def handle_starttag(self, tag, attrs): spec.loader.exec_module(asset_release) class AssetRelease(unittest.TestCase): + PRIVATE_NAMES = ( + 'await', 'break', 'case', 'catch', 'class', 'const', 'continue', + 'debugger', 'default', 'delete', 'do', 'else', 'enum', 'export', + 'extends', 'false', 'finally', 'for', 'function', 'if', 'import', + 'in', 'instanceof', 'new', 'null', 'return', 'super', 'switch', + 'this', 'throw', 'true', 'try', 'typeof', 'var', 'void', 'while', + 'with', 'yield', 'let', 'static', 'implements', 'interface', + 'package', 'private', 'protected', 'public', 'as', 'from', 'of', + 'using', 'async', 'get', 'set', 'x', 'α', '$x', '_x', + ) + def stage(self, root): subprocess.run(['git', '-C', str(root), 'add', '-A'], check=True, capture_output=True) @@ -513,6 +524,141 @@ def test_iteration_regex_refusals_preserve_all_files_and_index(self): self.assertIn(message, result.stderr) self.assertEqual(self.snapshot(root), before) + def test_private_keyword_targets_preserve_iteration_regex(self): + pattern = '/import("ghost.mjs?site-release=keep")/' + for name in self.PRIVATE_NAMES: + for header in ('for', 'for /*a*/ await /*b*/'): + for target in (f'this.#{name}', f'(this.#{name})', + f'this /*c*/ . /*d*/ #{name}', + f'this[#{name} in this]'): + source = (f'class C {{ #{name}; async run() {{ ' + f'{header} ({target} of {pattern}) {{}} }} }}') + with self.subTest(name=name, header=header, target=target): + try: + spans = asset_release.source_spans(source, '.mjs') + except ValueError as error: + self.fail(f'Private target was not recognized: {error}') + self.assertEqual(spans, []) + self.assertEqual(asset_release.transform( + source, '.mjs', asset_release.without_release), source) + + def test_private_keyword_divisions_discover_genuine_imports(self): + live = 'import("core.mjs")' + for name in self.PRIVATE_NAMES: + source = (f'class C {{ #{name}; run(obj) {{\n' + f'this.#{name} / {live} / 2;\n' + f'this?.#{name} / {live} / 2;\n' + f'for (let x = this.#{name} / {live} / 2; ; ) {{}}\n' + f'for (x of this.#{name} / {live} / 2) {{}}\n' + f'for (x in this.#{name} / {live} / 2) {{}}\n' + f'for (let x = (#{name} in obj); x; ) {{ {live}; }}\n' + '} }') + with self.subTest(name=name): + try: + spans = asset_release.source_spans(source, '.mjs') + except ValueError as error: + self.fail(f'Private division was not recognized: {error}') + self.assertEqual([source[a:b] for a, b in spans], ['core.mjs'] * 6) + self.assertEqual(asset_release.transform( + source, '.mjs', lambda value: value + '?site-release=new'), + source.replace('core.mjs', 'core.mjs?site-release=new')) + + def test_private_import_calls_are_not_module_syntax(self): + source = ('class C { #import() {} #if() {} run() {\n' + 'this.#import("ghost.mjs"); this?.#import("ghost.mjs");\n' + 'const t = `${this.#import("ghost.mjs")}`;\n' + 'this.#if(ok) / scale / import("core.mjs");\n' + 'import("core.mjs");\n' + '} }') + try: + spans = asset_release.source_spans(source, '.mjs') + except ValueError as error: + self.fail(f'Private call was treated as a module import: {error}') + self.assertEqual([source[a:b] for a, b in spans], ['core.mjs'] * 2) + self.assertEqual(asset_release.transform( + source, '.mjs', lambda value: value + '?site-release=new'), + source.replace('core.mjs', 'core.mjs?site-release=new')) + genuine_template = source.replace('${this.#import(', '${import(') + with self.assertRaisesRegex(ValueError, 'Import in template expression'): + asset_release.source_spans(genuine_template, '.mjs') + + def test_private_regex_cli_preserves_pattern_bytes_and_raw_index(self): + for name in ('in', 'return', 'const', 'of', 'x'): + for tracked in (False, True): + with self.subTest(name=name, tracked=tracked), TemporaryDirectory() as tmp: + root = Path(tmp) + subprocess.run(['git', 'init', '-q', str(root)], check=True, + capture_output=True) + site = root/'site'; site.mkdir() + source = (f'class C {{ #{name}; run() {{ for (this.#{name} ' + 'of /import("core.mjs")/) {} } }\n') + (site/'app.mjs').write_text(source) + if tracked: (site/'core.mjs').write_text('export const x = 1;\n') + self.stage(root) + before = self.snapshot(root) + index = (root/'.git/index').read_bytes() + token = None + for args in [('--check',), (), ('--check',), ()]: + result = self.run_tool(site, *args) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(result.stderr, '') + self.assertRegex(result.stdout, r'^[0-9a-f]{64}\n$') + if token is None: token = result.stdout + self.assertEqual(result.stdout, token) + self.assertEqual(self.snapshot(root), before) + self.assertEqual((root/'.git/index').read_bytes(), index) + + def test_private_calls_and_division_have_distinct_cli_dependencies(self): + for missing in (False, True): + with self.subTest(missing=missing), TemporaryDirectory() as tmp: + root = Path(tmp); site = self.fixture(root) + target = 'missing.mjs' if missing else 'core.mjs' + source = ('class C { #import() {} #return; run() {\n' + 'this.#import("ghost.mjs");\n' + f'this.#return / import("{target}") / 2;\n' + '} }\n') + (site/'app.mjs').write_text(source) + before = self.snapshot(root) + index = (root/'.git/index').read_bytes() + result = self.run_tool(site) + if missing: + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertEqual(result.stdout, '') + self.assertIn('missing/outside local asset missing.mjs', result.stderr) + self.assertEqual(self.snapshot(root), before) + else: + self.assertEqual(result.returncode, 0, result.stderr) + expected = source.replace('core.mjs', 'core.mjs?site-release=' + result.stdout.strip()) + self.assertEqual((site/'app.mjs').read_text(), expected) + after = self.snapshot(root) + for args in [('--check',), ()]: + again = self.run_tool(site, *args) + self.assertEqual(again.returncode, 0, again.stderr) + self.assertEqual(again.stdout, result.stdout) + self.assertEqual(self.snapshot(root), after) + self.assertEqual((root/'.git/index').read_bytes(), index) + + def test_unsupported_private_syntax_refuses_before_any_writes(self): + examples = [ + 'class C { #\\u0069n; run() { return this.#\\u0069n; } }', + 'class C { #; }', + 'class C { #/*comment*/in; }', + '#!/usr/bin/env node\nimport("./core.mjs");', + ] + for source in examples: + with self.subTest(source=source), TemporaryDirectory() as tmp: + root = Path(tmp); site = self.fixture(root) + (site/'app.mjs').write_text(source) + before = self.snapshot(root) + index = (root/'.git/index').read_bytes() + for args in [('--check',), ()]: + result = self.run_tool(site, *args) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertEqual(result.stdout, '') + self.assertIn('Unsupported JavaScript identifier syntax', result.stderr) + self.assertEqual(self.snapshot(root), before) + self.assertEqual((root/'.git/index').read_bytes(), index) + def test_untracked_dependency_and_non_git_site_are_refused(self): with TemporaryDirectory() as tmp: root = Path(tmp); site = self.fixture(root) diff --git a/tools/site_asset_release.py b/tools/site_asset_release.py index 889ef8f54..571632b9d 100644 --- a/tools/site_asset_release.py +++ b/tools/site_asset_release.py @@ -265,6 +265,14 @@ def _js_tokens(text, start=0, template_expression=False): else: raise ValueError('Unterminated template literal') token = ('template', text[begin:pos], begin, pos) + elif char == '#': + # PrivateIdentifier is one token, even when its name is a keyword. + # Escaped names and hashbangs remain outside this bounded grammar. + match = _NAME.match(text, pos + 1) + if match is None: + raise ValueError('Unsupported JavaScript identifier syntax') + pos = match.end() + token = ('private-name', text[begin:pos], begin, pos) elif char == '/': end = _regex_end(text, pos) property_word = (previous is not None and previous[0] == 'word' @@ -311,7 +319,7 @@ def _js_tokens(text, start=0, template_expression=False): property_name = previous is not None and previous[1] in ('.', '?.') previous_property = len(tokens) >= 2 and tokens[-2][1] in ('.', '?.') target_end = previous is not None and ( - previous[1] in (')', ']', '}') + previous[0] == 'private-name' or previous[1] in (')', ']', '}') or (previous[0] == 'word' and ( previous_property or (previous[1] not in _REGEX_PREFIX and previous[1] not in ('const', 'let', 'var', 'using'))))) From 056ec3163882a2a0d8021b51d5f739abcbf42766 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Wed, 7 Oct 2026 02:38:22 -0700 Subject: [PATCH 3/3] Recognize declaration roles and refuse unresolved lexical goals Use exact scalar-binding prefixes for contextual for-of scanning. Refuse unqualified await/yield slash and bare await/yield-of ambiguity uniformly without guessing loader mode. BQ42 R3 bounded author candidate; scientific effect NONE. --- tests/test_site_asset_release.py | 209 +++++++++++++++++++++++++++++++ tools/site_asset_release.py | 41 +++++- 2 files changed, 246 insertions(+), 4 deletions(-) diff --git a/tests/test_site_asset_release.py b/tests/test_site_asset_release.py index 4e700f352..2791b54f1 100644 --- a/tests/test_site_asset_release.py +++ b/tests/test_site_asset_release.py @@ -34,6 +34,24 @@ class AssetRelease(unittest.TestCase): 'package', 'private', 'protected', 'public', 'as', 'from', 'of', 'using', 'async', 'get', 'set', 'x', 'α', '$x', '_x', ) + LEXICAL_SLASH_CASES = ( + ('commonjs', 'var await = 1; await / import("core.mjs") / 2;'), + ('commonjs', 'var yield = 1; yield / import("core.mjs") / 2;'), + ('commonjs', 'for (var await = 1; await / import("core.mjs") / 2; ) {}'), + ('commonjs', 'for (var yield = 1; yield / import("core.mjs") / 2; ) {}'), + ('commonjs', 'function f() { var await = 1; await / import("core.mjs") / 2; }'), + ('commonjs', 'function f() { var yield = 1; yield / import("core.mjs") / 2; }'), + ('commonjs', 'var await = 1; await /= import("core.mjs");'), + ('commonjs', 'var yield = 1; yield /= import("core.mjs");'), + ('module', 'async function f() { return await /import("core.mjs")/; }'), + ('module', 'function* f() { yield /import("core.mjs")/; }'), + ('module', 'async function f() { return await /*gap*/ /import("core.mjs")/; }'), + ('module', 'function* f() { yield /*gap*/ /import("core.mjs")/; }'), + ('module', 'async function f() { return await //gap\n /import("core.mjs")/; }'), + ('module', 'function* f() { yield //gap\n /import("core.mjs")/.test("x"); }'), + ('module', 'async function f() { return `${await /import("core.mjs")/}`; }'), + ('module', 'function* f() { return `${yield /import("core.mjs")/}`; }'), + ) def stage(self, root): subprocess.run(['git', '-C', str(root), 'add', '-A'], check=True, @@ -659,6 +677,197 @@ def test_unsupported_private_syntax_refuses_before_any_writes(self): self.assertEqual(self.snapshot(root), before) self.assertEqual((root/'.git/index').read_bytes(), index) + def test_declaration_position_preserves_contextual_binding_regex(self): + pattern = '/import("ghost.mjs?site-release=keep")/.source' + module_bindings = [f'{decl} {name}' for decl in ('var', 'let', 'const') + for name in ('using', 'of', 'x', 'async')] + module_bindings += ['using using', 'using x', 'await using using', + 'await using of', 'await using x', 'using'] + cases = [('.mjs', f'{header} ({binding} of {pattern}) {{}}') + for header in ('for', 'for /*a*/ await /*b*/') + for binding in module_bindings] + cases += [('.js', f'for ({decl} {name} of {pattern}) {{}}') + for decl in ('var', 'let', 'const') for name in ('await', 'yield')] + cases += [('.js', f'for (var let of {pattern}) {{}}'), + ('.js', f'for ((await) of {pattern}) {{}}'), + ('.js', f'for ((yield) of {pattern}) {{}}'), + ('.mjs', f'for (const /*c*/ using /*d*/ of {pattern}) {{}}'), + ('.mjs', f'for (const\nusing of\n{pattern}) {{}}'), + ('.mjs', f'for (const x of xs) for (const using of {pattern}) {{}}'), + ('.mjs', f'for (const [using] of {pattern}) {{}}'), + ('.mjs', f'for (const {{using}} of {pattern}) {{}}')] + for suffix, source in cases: + with self.subTest(suffix=suffix, source=source): + self.assertEqual(asset_release.source_spans(source, suffix), []) + self.assertEqual(asset_release.transform( + source, suffix, asset_release.without_release), source) + + def test_declaration_prefix_does_not_hide_genuine_division_imports(self): + live = 'import("core.mjs")' + cases = [ + ('.mjs', f'for (const using = 1; using / {live} / 2; ) {{}}'), + ('.mjs', f'for (let using = of / {live} / 2; false; ) {{}}'), + ('.mjs', f'for (let x = using, of = 2; of / {live} / 2; ) {{}}'), + ('.mjs', f'for (let x = (of / {live} / 2); false; ) {{}}'), + ('.mjs', f'for (let x = {{using: of / {live} / 2}}; false; ) {{}}'), + ('.mjs', f'for (using / {live} / 2; false; ) {{}}'), + ('.mjs', f'for (using of of / {live} / 2) {{}}'), + ('.mjs', f'for (using using of of / {live} / 2) {{}}'), + ('.mjs', f'for (await using of of of / {live} / 2) {{}}'), + ('.mjs', f'for (const using of {live}) {{}}'), + ('.mjs', f'for (await using of of {live}) {{}}'), + ('.mjs', f'async function f() {{ for (let x = await of / {live} / 2; false; ) {{}} }}'), + ('.mjs', f'function* f() {{ for (let x = yield of / {live} / 2; false; ) {{}} }}'), + ('.mjs', f'async function f() {{ for (let x = await using / {live} / 2; false; ) {{}} }}'), + ('.js', f'for (var await = of / {live} / 2; false; ) {{}}'), + ('.js', f'for (var yield = of / {live} / 2; false; ) {{}}'), + ] + for suffix, source in cases: + with self.subTest(suffix=suffix, source=source): + spans = asset_release.source_spans(source, suffix) + self.assertEqual([source[a:b] for a, b in spans], ['core.mjs']) + self.assertEqual(asset_release.transform( + source, suffix, lambda value: value + '?site-release=new'), + source.replace('core.mjs', 'core.mjs?site-release=new')) + + def test_bare_await_yield_headers_refuse_both_valid_interpretations(self): + cases = [ + ('.js', 'for (await of /import("core.mjs")/) {}'), + ('.js', 'for (yield of /import("core.mjs")/) {}'), + ('.js', 'for (await of values) {}'), + ('.js', 'for (yield of values) {}'), + ('.mjs', 'async function f() { for (await of / import("core.mjs") / 2; false; ) {} }'), + ('.mjs', 'function* f() { for (yield of / import("core.mjs") / 2; false; ) {} }'), + ] + for suffix, source in cases: + with self.subTest(suffix=suffix, source=source): + with self.assertRaisesRegex(ValueError, 'Ambiguous await/yield for header'): + asset_release.source_spans(source, suffix) + + def test_binding_position_cli_preserves_tracked_and_missing_regex_targets(self): + cases = [ + ('.mjs', 'for (const using of /import("core.mjs")/) {}\n'), + ('.mjs', 'for await (let using of /import("core.mjs")/) {}\n'), + ('.mjs', 'for (using of /import("core.mjs")/) {}\n'), + ('.mjs', 'for (await using of of /import("core.mjs")/) {}\n'), + ('.js', 'for (var await of /import("core.mjs")/) {}\n'), + ] + for suffix, source in cases: + for tracked in (False, True): + with self.subTest(source=source, tracked=tracked), TemporaryDirectory() as tmp: + root = Path(tmp) + subprocess.run(['git', 'init', '-q', str(root)], check=True, + capture_output=True) + site = root/'site'; site.mkdir() + (site/('app' + suffix)).write_text(source) + if tracked: (site/'core.mjs').write_text('export const x = 1;\n') + self.stage(root) + before = self.snapshot(root); index = (root/'.git/index').read_bytes() + token = None + for args in [('--check',), (), ('--check',), ()]: + result = self.run_tool(site, *args) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(result.stderr, '') + self.assertRegex(result.stdout, r'^[0-9a-f]{64}\n$') + if token is None: token = result.stdout + self.assertEqual(result.stdout, token) + self.assertEqual(self.snapshot(root), before) + self.assertEqual((root/'.git/index').read_bytes(), index) + + def test_ambiguous_binding_cli_refuses_before_all_file_and_index_writes(self): + cases = [ + ('.js', 'for (await of /import("core.mjs")/) {}'), + ('.js', 'for (yield of /import("missing.mjs")/) {}'), + ('.mjs', 'async function f() { for (await of / import("core.mjs") / 2; false; ) {} }'), + ('.mjs', 'function* f() { for (yield of / import("core.mjs") / 2; false; ) {} }'), + ] + for suffix, source in cases: + with self.subTest(source=source), TemporaryDirectory() as tmp: + root = Path(tmp); site = self.fixture(root) + # Earlier files have genuine pending updates when the final file refuses. + (site/('z-ambiguous' + suffix)).write_text(source) + self.stage(root) + before = self.snapshot(root); index = (root/'.git/index').read_bytes() + for args in [('--check',), ()]: + result = self.run_tool(site, *args) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertEqual(result.stdout, '') + self.assertIn('Ambiguous await/yield for header', result.stderr) + self.assertEqual(self.snapshot(root), before) + self.assertEqual((root/'.git/index').read_bytes(), index) + + def test_declared_binding_regex_and_real_import_cli_remain_distinct(self): + for missing in (False, True): + with self.subTest(missing=missing), TemporaryDirectory() as tmp: + root = Path(tmp); site = self.fixture(root) + target = 'missing.mjs' if missing else 'core.mjs' + source = ('for (const using of /import("ghost.mjs")/.source) { ' + f'import("{target}"); }}\n') + (site/'app.mjs').write_text(source) + before = self.snapshot(root); index = (root/'.git/index').read_bytes() + result = self.run_tool(site) + if missing: + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertEqual(result.stdout, '') + self.assertIn('missing/outside local asset missing.mjs', result.stderr) + self.assertEqual(self.snapshot(root), before) + else: + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((site/'app.mjs').read_text(), source.replace( + 'core.mjs', 'core.mjs?site-release=' + result.stdout.strip())) + after = self.snapshot(root) + for args in [('--check',), ()]: + again = self.run_tool(site, *args) + self.assertEqual(again.returncode, 0, again.stderr) + self.assertEqual(again.stdout, result.stdout) + self.assertEqual(self.snapshot(root), after) + self.assertEqual((root/'.git/index').read_bytes(), index) + + def test_unqualified_await_yield_slash_refuses_without_mode_inference(self): + for mode, source in self.LEXICAL_SLASH_CASES: + # The checked fixture mode is evidence, not an input to the scanner. + for suffix in ('.js', '.mjs'): + with self.subTest(mode=mode, suffix=suffix, source=source): + with self.assertRaisesRegex(ValueError, 'Ambiguous await/yield regex/division'): + asset_release.source_spans(source, suffix) + + def test_lexical_goal_refusal_preserves_pending_files_and_raw_index(self): + for mode, source in self.LEXICAL_SLASH_CASES: + with self.subTest(mode=mode, source=source), TemporaryDirectory() as tmp: + root = Path(tmp); site = self.fixture(root) + suffix = '.js' if mode == 'commonjs' else '.mjs' + (site/('z-ambiguous' + suffix)).write_text(source) + self.stage(root) + before = self.snapshot(root); index = (root/'.git/index').read_bytes() + for args in [('--check',), ()]: + result = self.run_tool(site, *args) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertEqual(result.stdout, '') + self.assertIn('Ambiguous await/yield regex/division', result.stderr) + self.assertEqual(self.snapshot(root), before) + self.assertEqual((root/'.git/index').read_bytes(), index) + + def test_unambiguous_await_yield_and_opaque_words_keep_module_spans(self): + cases = [ + ('async function f() { return await import("core.mjs"); }', ['core.mjs']), + ('function* f() { yield import("core.mjs"); }', ['core.mjs']), + ('async function f() { return await (/import("ghost.mjs")/); }', []), + ('function* f() { yield (/import("ghost.mjs")/); yield* /import("ghost.mjs")/.source; }', []), + ('async function f() { return `${await (/import("ghost.mjs")/)}`; }', []), + ('obj.await / import("core.mjs") / 2;\nobj?.yield / import("core.mjs") / 2;', + ['core.mjs', 'core.mjs']), + ('class C { #await; #yield; f() {\nthis.#await / import("core.mjs") / 2;\n' + 'this?.#yield / import("core.mjs") / 2;\n} }', ['core.mjs', 'core.mjs']), + ('// await / import("ghost.mjs") /\n/* yield / import("ghost.mjs") / */\n' + 'const a = \'await / import("ghost.mjs") /\';\n' + 'const b = `yield / import("ghost.mjs") /`;\n' + 'const c = /await[/]import("ghost.mjs")/; import("core.mjs");', ['core.mjs']), + ] + for source, expected in cases: + with self.subTest(source=source): + spans = asset_release.source_spans(source, '.mjs') + self.assertEqual([source[a:b] for a, b in spans], expected) + def test_untracked_dependency_and_non_git_site_are_refused(self): with TemporaryDirectory() as tmp: root = Path(tmp); site = self.fixture(root) diff --git a/tools/site_asset_release.py b/tools/site_asset_release.py index 571632b9d..ae4f4a14c 100644 --- a/tools/site_asset_release.py +++ b/tools/site_asset_release.py @@ -6,6 +6,8 @@ Only recognized live URL spans are normalized/rewritten. This bounded stdlib scanner refuses unsupported escaped URLs and ambiguous JavaScript contexts before any writes; it is not a complete JavaScript parser. +Unqualified await/yield slash and bare for-of heads are refused: a filename +suffix does not establish the JavaScript loader mode or lexical goal. """ import argparse import hashlib @@ -225,6 +227,27 @@ def _regex_end(text, start): return None +def _for_binding_prefix(tokens, start): + """Recognize only the complete short scalar-binding prefix before `of`.""" + if len(tokens) - start > 3: + return None + head = tokens[start:] + if any(token[0] != 'word' for token in head): + return None + words = tuple(token[1] for token in head) + if words in (('await',), ('yield',)): + return 'ambiguous' + if words in (('var',), ('let',), ('const',), ('await', 'using')): + return 'name-needed' + # `using of` is the bare using target; `await using of of` instead + # names its binding `of` before the second, separating `of`. + if (words == ('using',) + or (len(words) == 2 and words[0] in ('var', 'let', 'const', 'using')) + or (len(words) == 3 and words[:2] == ('await', 'using'))): + return 'complete' + return None + + def _js_tokens(text, start=0, template_expression=False): # Tokens are (kind, spelling, start, end). tokens = [] @@ -274,10 +297,14 @@ def _js_tokens(text, start=0, template_expression=False): pos = match.end() token = ('private-name', text[begin:pos], begin, pos) elif char == '/': - end = _regex_end(text, pos) property_word = (previous is not None and previous[0] == 'word' and len(tokens) >= 2 and tokens[-2][1] in ('.', '?.')) + if (previous is not None and previous[0] == 'word' + and previous[1] in ('await', 'yield') and not property_word): + raise ValueError('Ambiguous await/yield regex/division requires ' + 'a full JavaScript parser') + end = _regex_end(text, pos) prefix = (previous is None or previous[0] in ('control-close', 'for-of') or (previous[1] in _REGEX_PREFIX and not property_word)) ambiguous = previous is not None and previous[1] in (')', '}') @@ -325,9 +352,14 @@ def _js_tokens(text, start=0, template_expression=False): and previous[1] not in ('const', 'let', 'var', 'using'))))) if token[1] == ';' or (token[1] == 'in' and not property_name): parens[-1]['for-binding'] = False - elif token[1] == 'of' and not property_name and target_end: - token = ('for-of', 'of', begin, pos) - parens[-1]['for-binding'] = False + elif token[1] == 'of' and not property_name: + binding = _for_binding_prefix(tokens, parens[-1]['start']) + if binding == 'ambiguous': + raise ValueError('Ambiguous await/yield for header requires ' + 'a full JavaScript parser') + if binding == 'complete' or (binding != 'name-needed' and target_end): + token = ('for-of', 'of', begin, pos) + parens[-1]['for-binding'] = False if token[1] == '(': head = len(tokens) - 1 @@ -339,6 +371,7 @@ def _js_tokens(text, start=0, template_expression=False): and (head == 0 or tokens[head - 1][1] not in ('.', '?.'))) parens.append({'control': control, 'for-binding': control and tokens[head][1] == 'for', + 'start': len(tokens) + 1, 'depth': (braces, brackets)}) elif token[1] == ')': control = parens.pop()['control'] if parens else False