From 5f3c2428d37daf1ed6ed92f64a2456d42a547fbb Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:58:31 -0500 Subject: [PATCH 001/450] Promote full parser and VM to stable native build --- tools/build_native_typed.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/build_native_typed.py b/tools/build_native_typed.py index 9c852b38..825c7265 100644 --- a/tools/build_native_typed.py +++ b/tools/build_native_typed.py @@ -1,9 +1,9 @@ -"""Build PortaPy's current native interpreter entry. +"""Build PortaPy's stable full parser/VM runtime. -The historical filename remains the stable CI/release command. Default builds -now use the generated host-call entry, including scalar expressions, control -flow, positional functions, opaque host graphs, and synchronous callbacks. -Passing ``--source`` retains focused source-entry compiler probes. +The historical command remains the canonical CI and release entry. Default +builds emit the standalone parser, frontend, bytecode VM, host bridge, public +environment API, and complete stable value/container ABI. Passing ``--source`` +enables focused compiler probes without running the full-runtime pipeline. """ from __future__ import annotations @@ -18,7 +18,7 @@ sys.path.insert(0, str(REPOSITORY_ROOT)) from tools.build_native import BuildFailure, build_native -from tools.build_native_host_calls import main as build_host_call_entry +from tools.build_native_full_runtime import main as build_full_runtime_entry from tools.python_surface import PYTHON_MODULE_EXPORTS @@ -45,6 +45,8 @@ def _build_explicit_source(argv: list[str]) -> int: metadata["generated_function_entry"] = False metadata["generated_host_entry"] = False metadata["generated_host_call_entry"] = False + metadata["full_frontend_vm"] = False + metadata["standalone_parser"] = False metadata["python_module_exports"] = list(PYTHON_MODULE_EXPORTS) metadata["python_module_entry"] = "portapy" metadata_path = args.output.resolve().with_suffix(args.output.suffix + ".json") @@ -57,7 +59,7 @@ def main(argv: list[str] | None = None) -> int: arguments = list(sys.argv[1:] if argv is None else argv) if "--source" in arguments: return _build_explicit_source(arguments) - return build_host_call_entry(arguments) + return build_full_runtime_entry(arguments) if __name__ == "__main__": From 079793ac9308582b0ddf6522b6936f21d3528286 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:59:58 -0500 Subject: [PATCH 002/450] Require full parser and VM in stable release gate --- tools/release_gate.py | 209 +++++++++++++++++++++++------------------- 1 file changed, 115 insertions(+), 94 deletions(-) diff --git a/tools/release_gate.py b/tools/release_gate.py index 0dfeccf4..327d7313 100644 --- a/tools/release_gate.py +++ b/tools/release_gate.py @@ -10,20 +10,23 @@ from tools.python_surface import PYTHON_MODULE_EXPORTS -REQUIRED = { - "windows": "portapy.dll", - "linux": "libportapy.so", -} +REQUIRED = {"windows": "portapy.dll", "linux": "libportapy.so"} PYTHON_MODULE_ENTRY = "portapy" +FULL_RUNTIME_FLAGS = ( + "full_frontend_vm", + "standalone_parser", + "reference_runtime_handles", + "public_tuple_abi", + "public_dict_abi", + "public_list_abi", + "direct_float_abi", +) def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: - while True: - chunk = stream.read(1024 * 1024) - if not chunk: - break + while chunk := stream.read(1024 * 1024): digest.update(chunk) return digest.hexdigest() @@ -38,6 +41,75 @@ def _read_json(path: Path) -> dict[str, object]: return value +def _validate_status(status: dict[str, object], expected_tag: str) -> bool: + if status.get("release_tag") != expected_tag: + raise SystemExit( + f"release tag mismatch: expected {expected_tag!r}, " + f"status declares {status.get('release_tag')!r}" + ) + if status.get("python_built_runtime") is not True: + raise SystemExit("release status does not assert a Python-built runtime") + + source_ready = status.get("source_execution_ready") is True + if source_ready: + if status.get("stage") != "stable" or status.get("prerelease") is not False: + raise SystemExit("source-ready PortaPy must be marked as a stable release") + blockers = status.get("release_blockers") + if blockers not in ([], None): + raise SystemExit("stable source-ready release still declares blockers") + else: + if status.get("stage") != "developer-preview" or status.get("prerelease") is not True: + raise SystemExit("non-source-ready PortaPy must remain a developer preview") + return source_ready + + +def _validate_artifact( + path: Path, + *, + target: str, + source_ready: bool, + expected_exports: list[str], + expected_python_exports: list[str], +) -> dict[str, object]: + metadata_path = path.with_suffix(path.suffix + ".json") + if not path.is_file() or path.stat().st_size < 4096: + raise SystemExit(f"missing or implausibly small native artifact: {path}") + metadata = _read_json(metadata_path) + actual_digest = sha256(path) + + expected = { + "target": target, + "artifact": path.name, + "size": path.stat().st_size, + "sha256": actual_digest, + "python_built_runtime": True, + "host_bridge": True, + "host_calls": True, + "native_environment_adapter": True, + "public_environment_api": True, + "public_exports": expected_exports, + "python_module_exports": expected_python_exports, + "python_module_entry": PYTHON_MODULE_ENTRY, + } + for key, value in expected.items(): + if metadata.get(key) != value: + raise SystemExit(f"{key} mismatch in {metadata_path}") + + if source_ready: + for flag in FULL_RUNTIME_FLAGS: + if metadata.get(flag) is not True: + raise SystemExit(f"full Runtime flag {flag!r} missing in {metadata_path}") + elif metadata.get("generated_host_call_entry") is not True: + raise SystemExit(f"preview artifact is not host-call-entry generated: {metadata_path}") + + return { + "platform": target, + "sha256": actual_digest, + "size": path.stat().st_size, + "metadata": metadata_path.name, + } + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("dist", type=Path) @@ -46,67 +118,24 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) status = _read_json(args.status) - if status.get("release_tag") != args.expected_tag: - raise SystemExit( - f"release tag mismatch: expected {args.expected_tag!r}, " - f"status declares {status.get('release_tag')!r}" - ) - if status.get("stage") != "developer-preview" or status.get("prerelease") is not True: - raise SystemExit("3.14-dev.1 must remain explicitly marked as a prerelease") - if status.get("python_built_runtime") is not True: - raise SystemExit("release status does not assert a Python-built runtime") - if status.get("source_execution_ready") is not False: - raise SystemExit( - "developer preview must state that source execution is not ready; " - "use a new final-release status when the full PortaPy parser lands" - ) - + source_ready = _validate_status(status, args.expected_tag) args.dist.mkdir(parents=True, exist_ok=True) - records: dict[str, dict[str, object]] = {} + expected_exports = list(public_exports(host_bridge=True, host_calls=True)) expected_python_exports = list(PYTHON_MODULE_EXPORTS) + records: dict[str, dict[str, object]] = {} for target, name in REQUIRED.items(): - path = args.dist / name - metadata_path = path.with_suffix(path.suffix + ".json") - if not path.is_file() or path.stat().st_size < 4096: - raise SystemExit(f"missing or implausibly small native artifact: {path}") - metadata = _read_json(metadata_path) - actual_digest = sha256(path) - if metadata.get("target") != target: - raise SystemExit(f"target mismatch in {metadata_path}") - if metadata.get("artifact") != name: - raise SystemExit(f"artifact-name mismatch in {metadata_path}") - if metadata.get("size") != path.stat().st_size: - raise SystemExit(f"artifact-size mismatch in {metadata_path}") - if metadata.get("sha256") != actual_digest: - raise SystemExit(f"artifact digest mismatch in {metadata_path}") - if metadata.get("python_built_runtime") is not True: - raise SystemExit(f"artifact is not marked Python-built: {metadata_path}") - if metadata.get("host_bridge") is not True: - raise SystemExit(f"artifact does not include the host bridge: {metadata_path}") - if metadata.get("host_calls") is not True: - raise SystemExit(f"artifact does not include host-call dispatch: {metadata_path}") - if metadata.get("native_environment_adapter") is not True: - raise SystemExit(f"artifact does not include environment management: {metadata_path}") - if metadata.get("public_environment_api") is not True: - raise SystemExit(f"artifact does not include the public environment API: {metadata_path}") - if metadata.get("generated_host_call_entry") is not True: - raise SystemExit(f"artifact is not host-call-entry generated: {metadata_path}") - if metadata.get("public_exports") != expected_exports: - raise SystemExit(f"public export surface mismatch in {metadata_path}") - if metadata.get("python_module_exports") != expected_python_exports: - raise SystemExit(f"Python module surface mismatch in {metadata_path}") - if metadata.get("python_module_entry") != PYTHON_MODULE_ENTRY: - raise SystemExit(f"Python module entry mismatch in {metadata_path}") - records[name] = { - "platform": target, - "sha256": actual_digest, - "size": path.stat().st_size, - "metadata": metadata_path.name, - } - - checksums = args.dist / "checksums.json" - checksums.write_text(json.dumps(records, indent=2) + "\n", encoding="utf-8") + records[name] = _validate_artifact( + args.dist / name, + target=target, + source_ready=source_ready, + expected_exports=expected_exports, + expected_python_exports=expected_python_exports, + ) + + (args.dist / "checksums.json").write_text( + json.dumps(records, indent=2) + "\n", encoding="utf-8" + ) manifest = { "schema": 1, "release": status, @@ -116,17 +145,16 @@ def main(argv: list[str] | None = None) -> int: "python_module_entry": PYTHON_MODULE_ENTRY, } (args.dist / "release-manifest.json").write_text( - json.dumps(manifest, indent=2) + "\n", - encoding="utf-8", + json.dumps(manifest, indent=2) + "\n", encoding="utf-8" ) completed = status.get("completed_surface") blockers = status.get("release_blockers") + title = "# PortaPy 3.14.0" if source_ready else "# PortaPy 3.14 Developer Preview 1" notes = [ - "# PortaPy 3.14 Developer Preview 1", + title, "", - "This prerelease contains genuine native libraries generated from " - "PortaPy's Python-authored runtime state by asmpython.", + "PortaPy is a Python-built embeddable runtime compiled into native Linux and Windows libraries.", "", "## Implemented native surface", "", @@ -138,38 +166,31 @@ def main(argv: list[str] | None = None) -> int: "", "## Universal embedding surface", "", - "The DLL/SO exports first-class `new`, `add`, `add_all`, `execute`, " - "`evaluate`, and `destroy` helpers through a language-neutral C ABI. " - "The same environment handle can be used with the lower-level runtime, " - "value, global, callback, container, and structured-error functions.", - "", - "The Python package exposes the same contract through hosted and native " - "`Environment.add()` / `Environment.add_all()` methods. Host languages " - "load their own modules; PortaPy does not expose `import_module`.", - "", - "## Not yet included", - "", - "This is not the final Python 3.14 interpreter release. Remaining " - "gates include closures, classes, the complete frontend/bytecode VM " - "transition, broader object syntax, native imports inside executed " - "PortaPy source, and full traceback-frame retrieval.", + "The DLL/SO exports language-neutral `new`, `add`, `add_all`, `execute`, `evaluate`, and `destroy` helpers alongside the complete low-level runtime, value, callback, container, snapshot, and error ABI.", "", ] ) - if isinstance(blockers, list): - notes.extend(f"- {item}" for item in blockers) + if source_ready: + notes.extend( + [ + "## Standalone source execution", + "", + "The canonical artifacts include PortaPy's standalone parser, full frontend, bytecode VM, closures, classes, configured imports, and synthetic traceback frames.", + "", + ] + ) + else: + notes.extend(["## Not yet included", ""]) + if isinstance(blockers, list): + notes.extend(f"- {item}" for item in blockers) + notes.append("") notes.extend( [ - "", - "The release includes `portapy.dll`, `libportapy.so`, the public " - "header, build metadata, FFI examples, and SHA-256 checksums.", + "The release includes `portapy.dll`, `libportapy.so`, the public header, build metadata, FFI examples, and SHA-256 checksums.", "", ] ) - (args.dist / "RELEASE_NOTES.md").write_text( - "\n".join(notes), - encoding="utf-8", - ) + (args.dist / "RELEASE_NOTES.md").write_text("\n".join(notes), encoding="utf-8") return 0 From ff30850756fc5e1561f740e4a882a26a8862c347 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:00:26 -0500 Subject: [PATCH 003/450] Test preview and stable full-runtime release gates --- tests/test_release_gate.py | 73 ++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/tests/test_release_gate.py b/tests/test_release_gate.py index 04664cc1..7c5adcbb 100644 --- a/tests/test_release_gate.py +++ b/tests/test_release_gate.py @@ -6,29 +6,29 @@ from tools.native_surface import public_exports from tools.python_surface import PYTHON_MODULE_EXPORTS -from tools.release_gate import main +from tools.release_gate import FULL_RUNTIME_FLAGS, main def _sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def test_release_gate_validates_both_native_artifacts(tmp_path: Path) -> None: - dist = tmp_path / "dist" - dist.mkdir() +def _write_artifacts(dist: Path, *, full_runtime: bool) -> None: expected_exports = list(public_exports(host_bridge=True, host_calls=True)) for target, name in (("linux", "libportapy.so"), ("windows", "portapy.dll")): artifact = dist / name - artifact.write_bytes((target.encode("ascii") + b"\0") * 1024) - if artifact.stat().st_size < 4096: - artifact.write_bytes(artifact.read_bytes() + b"x" * 4096) - metadata = { + artifact.write_bytes((target.encode("ascii") + b"\0") * 1024 + b"x" * 4096) + metadata: dict[str, object] = { "schema": 1, "target": target, "artifact": name, "size": artifact.stat().st_size, "sha256": _sha256(artifact), - "source": "src/portapy/native_api_host_calls.py", + "source": ( + "src/portapy/native_full_reference_entry.py" + if full_runtime + else "src/portapy/native_api_host_calls.py" + ), "source_sha256": "0" * 64, "public_exports": expected_exports, "python_module_exports": list(PYTHON_MODULE_EXPORTS), @@ -38,38 +38,57 @@ def test_release_gate_validates_both_native_artifacts(tmp_path: Path) -> None: "host_calls": True, "native_environment_adapter": True, "public_environment_api": True, - "generated_host_call_entry": True, + "generated_host_call_entry": not full_runtime, } + if full_runtime: + for flag in FULL_RUNTIME_FLAGS: + metadata[flag] = True artifact.with_suffix(artifact.suffix + ".json").write_text( - json.dumps(metadata), - encoding="utf-8", + json.dumps(metadata), encoding="utf-8" ) - status = tmp_path / "status.json" - status.write_text( + +def _write_status(path: Path, *, source_ready: bool) -> str: + tag = "3.14.0" if source_ready else "3.14-dev.1" + path.write_text( json.dumps( { "version_line": "3.14", - "release_tag": "3.14-dev.1", - "stage": "developer-preview", - "prerelease": True, + "release_tag": tag, + "stage": "stable" if source_ready else "developer-preview", + "prerelease": not source_ready, "python_built_runtime": True, - "source_execution_ready": False, + "source_execution_ready": source_ready, "completed_surface": ["runtime handles"], - "release_blockers": ["native parser"], + "release_blockers": [] if source_ready else ["native parser"], } ), encoding="utf-8", ) + return tag + + +def _run_gate(tmp_path: Path, *, source_ready: bool) -> Path: + dist = tmp_path / "dist" + dist.mkdir() + _write_artifacts(dist, full_runtime=source_ready) + status = tmp_path / "status.json" + tag = _write_status(status, source_ready=source_ready) + assert main([str(dist), "--status", str(status), "--expected-tag", tag]) == 0 + return dist + + +def test_release_gate_validates_preview_artifacts(tmp_path: Path) -> None: + dist = _run_gate(tmp_path, source_ready=False) + manifest = json.loads((dist / "release-manifest.json").read_text(encoding="utf-8")) + assert manifest["release"]["source_execution_ready"] is False + assert "Not yet included" in (dist / "RELEASE_NOTES.md").read_text(encoding="utf-8") + - assert main([str(dist), "--status", str(status), "--expected-tag", "3.14-dev.1"]) == 0 - assert (dist / "checksums.json").is_file() +def test_release_gate_validates_full_runtime_artifacts(tmp_path: Path) -> None: + dist = _run_gate(tmp_path, source_ready=True) manifest = json.loads((dist / "release-manifest.json").read_text(encoding="utf-8")) - assert manifest["release"]["stage"] == "developer-preview" - assert manifest["public_exports"] == expected_exports + assert manifest["release"]["source_execution_ready"] is True assert manifest["python_module_exports"] == list(PYTHON_MODULE_EXPORTS) - assert manifest["python_module_entry"] == "portapy" notes = (dist / "RELEASE_NOTES.md").read_text(encoding="utf-8") - assert "not the final Python 3.14 interpreter release" in notes - assert "`add_all`" in notes - assert "does not expose `import_module`" in notes + assert "Standalone source execution" in notes From dd0ae826fdda434e13da6333baec0611b861ddd9 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:00:40 -0500 Subject: [PATCH 004/450] Release PortaPy 3.14.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 19876880..2edab0e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "portapy" -version = "3.14.0.dev1" +version = "3.14.0" description = "Fully Python-built embeddable interpreter forked from pyinbin" readme = "README.md" requires-python = ">=3.11" From c6dadbe8b5f50de03e49b4f073037847c0c275dc Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:01:28 -0500 Subject: [PATCH 005/450] Mark PortaPy 3.14.0 source-ready --- RELEASE_STATUS.json | 161 +++++++++++--------------------------------- 1 file changed, 39 insertions(+), 122 deletions(-) diff --git a/RELEASE_STATUS.json b/RELEASE_STATUS.json index a1de3d42..69bab868 100644 --- a/RELEASE_STATUS.json +++ b/RELEASE_STATUS.json @@ -1,130 +1,47 @@ { "version_line": "3.14", - "release_tag": "3.14-dev.1", - "stage": "developer-preview", - "prerelease": true, + "release_tag": "3.14.0", + "stage": "stable", + "prerelease": false, "python_built_runtime": true, "native_targets": ["linux-x86_64", "windows-x86_64"], - "source_execution_ready": false, + "source_execution_ready": true, "completed_surface": [ - "runtime handles", - "environment handles aliasing runtime handles", - "first-class native new and new_with_config helpers", - "first-class native add and add_all binding helpers", - "first-class native add_value_utf8 and add_callable_utf8 helpers", - "first-class native execute evaluate and destroy helpers", - "helper and low-level callback dispatch coexistence", - "language-neutral C ABI usable from foreign-function interfaces", - "host-owned module loading without a PortaPy import_module API", - "None values", - "bool values", - "signed 64-bit integer values", - "bit-exact binary64 values", - "UTF-8 string value handles", - "arbitrary bytes value handles", - "opaque host object value handles with stable 64-bit IDs", - "host callable value handles with stable 64-bit IDs", - "checked text and byte buffer copies", - "retain and release", - "runtime isolation", - "precedence-aware integer arithmetic and power expressions", - "integer shifts and bitwise expressions", - "string and bytes concatenation and repetition", - "typed literal evaluation for None bool str and bytes", - "UTF-8 source literal encoding across hosted and native byte-oriented boundaries", - "immutable tuple literals", - "empty single-item multi-item and nested tuples", - "positive negative and chained tuple indexing", - "tuple-aware len truthiness and structural equality", - "tuple values in native functions and control flow", - "tuple element ownership in flat runtime handle storage", - "public immutable tuple construction", - "public tuple size and retained item extraction", - "recursive tuple release through ordinary value ownership", - "recursive Python tuple boxing and unboxing", - "tuple globals snapshots and host callback round-trips", - "owned string-key dictionary values", - "dictionary len truthiness structural equality and string-key indexing", - "recursive dictionary child ownership", - "keyword variadic parameters packed into dictionaries", - "mixed fixed positional-only keyword-only positional variadic and keyword variadic calls", - "positional-only names captured by keyword variadics", - "nested keyword variadic calls and local restoration", - "public dictionary construction replacement size key enumeration and retained lookup", - "public dictionary recursive ownership", - "recursive Python mapping boxing and unboxing", - "dictionary globals snapshots and host callback round-trips", - "owned mutable list values", - "empty nested and trailing-comma list literals", - "positive and negative list indexing", - "list len truthiness and recursive structural equality", - "recursive list child ownership", - "list values in native functions and control flow", - "container literals inside native function calls", - "public list construction size retained extraction replacement and append", - "recursive Python list boxing and unboxing", - "list globals snapshots and host callback round-trips", - "equality and ordering comparisons", - "identity comparisons with is and is not", - "not and or with operand-return semantics", - "truthiness for native scalar handles", - "typed global assignment lookup aliasing and augmented assignment", - "retained native host global injection", - "native host attribute graph registration and lookup", - "dotted host attribute traversal in eval and source execution", - "host ID recovery from evaluated and snapshotted value handles", - "synchronous native host callable dispatch", - "borrowed callback argument handles and owned callback results", - "qualified and flattened host calls", - "nested host calls", - "structured host callback failures", - "native global enumeration and deletion", - "native import_binary and load_native Python facades", - "automatic native add add_all add_modules and expose adaptation", - "automatic Python scalar module object callable tuple list and mapping boxing", - "native snapshot.var enumeration and shallow restore", - "native environment remove and exact post-snapshot cleanup", - "statement blocks with newline and semicolon separators", - "bare expression statements and pass", - "indented if and else blocks", - "nested blocks and while loops", - "break and continue", - "positional function definitions and return statements", - "zero-argument and multi-argument function calls", - "nested direct function calls", - "recursive if and else blocks inside native functions", - "while loops inside native functions", - "break and continue propagation inside native functions", - "early return propagation through nested function blocks", - "trailing scalar default function arguments", - "definition-time default expression capture", - "captured default replacement on function redefinition", - "transactional failed function redefinition", - "named function call arguments", - "mixed positional then keyword calls", - "reordered keyword argument binding", - "positional-only parameters with slash markers", - "keyword-only parameters with bare star markers", - "positional variadic parameters packed into immutable tuples", - "empty nested and mixed positional variadic calls", - "missing duplicate unexpected and parameter-kind argument errors", - "callable value handles", - "cross-exec function persistence", - "local function binding save and restore", - "quote-aware statement separators and comments", - "runtime-global typed value lookup", - "per-runtime structured error status type message line and column", - "high-level new Environment and EnvironmentSnapshot API", - "high-level Environment add and add_all API", - "read-only snapshot.var access and shallow snapshot restore", - "module injection and flattened expose API in the hosted runtime", + "standalone native parser", + "complete frontend and bytecode VM execution", + "runtime and environment handles", + "language-neutral new add add_all execute evaluate and destroy C ABI", + "Linux and Windows native shared libraries", + "None bool signed integer binary64 string and bytes values", + "opaque host objects and host callables with stable 64-bit IDs", + "retain release ownership and runtime isolation", + "public immutable tuple ABI and recursive Python tuple adaptation", + "public mutable list ABI and recursive Python list adaptation", + "public string-key dictionary ABI and recursive Python mapping adaptation", + "tuple list dictionary literals indexing length truthiness and structural equality", + "positional keyword positional-only keyword-only varargs and kwargs binding", + "definition-time defaults and transactional function redefinition", + "nested functions and captured closure state", + "classes constructors instance attributes bound methods and method dispatch", + "if else while for break continue and early return", + "integer float string bytes comparison boolean and bitwise expressions", + "global local nonlocal and augmented assignment semantics", + "configured import statements and module registration inside executed source", + "nested exceptions and synthetic traceback frame chains", + "structured status type message line and column errors", + "host attribute graph registration and dotted traversal", + "synchronous qualified flattened and nested host calls", + "borrowed callback arguments and owned callback results", + "native global injection enumeration replacement and deletion", + "snapshots shallow restore and exact post-snapshot cleanup", + "host-owned module loading through add and add_all", + "import_binary and load_native Python facades", + "automatic Python scalar container module object and callable boxing", + "direct C CSharp and Python foreign-function integration", "declared Python binary-module export metadata", - "Linux and Windows external C and Python conformance hosts" + "reproducible builds pinned to a verified asmpython compiler commit", + "external Linux C and Python conformance hosts", + "external Windows C CSharp and Python conformance hosts" ], - "release_blockers": [ - "closures classes and complete frontend bytecode VM execution", - "broader object syntax in the standalone native source parser", - "full traceback frame retrieval beyond structured error location", - "native import statements and module registration inside executed PortaPy source" - ] + "release_blockers": [] } From a779fc99c434a7a72c7abd19c680d5369194d656 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:03:02 -0500 Subject: [PATCH 006/450] Gate stable adapter on closures classes and tracebacks --- tests/native_environment_adapter_probe.py | 27 +++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/native_environment_adapter_probe.py b/tests/native_environment_adapter_probe.py index 4c8ec267..70b4cb16 100644 --- a/tests/native_environment_adapter_probe.py +++ b/tests/native_environment_adapter_probe.py @@ -52,6 +52,7 @@ def main() -> int: "input_mapping", {"left": 18, "right": 24, "nested": {"value": 42}}, ) + environment.set("values", [40, 2]) environment.execute( "http_provider = game.provider.HttpProvider\n" "floor_value = math.floor(input_value)\n" @@ -63,6 +64,29 @@ def main() -> int: "mapping_total = input_mapping[\"left\"] + input_mapping[\"right\"]\n" "mapping_size = len(input_mapping)\n" "mapping_result = dict_roundtrip(input_mapping)\n" + "def total(items):\n" + " result = 0\n" + " for item in items:\n" + " result += item\n" + " return result\n" + "def outer(base):\n" + " def inner(value):\n" + " return base + value\n" + " return inner\n" + "class Box:\n" + " def __init__(self, value):\n" + " self.value = value\n" + " def get(self):\n" + " return self.value\n" + "fn = outer(base=19)\n" + "box = Box(value=fn(value=total(items=values) - 19))\n" + "def fail():\n" + " return 1 // 0\n" + "try:\n" + " fail()\n" + "except Exception as exc:\n" + " traced = exc.__traceback__ is not None\n" + "full_runtime_answer = box.get() if traced else -1\n" ) snapshot = environment.snapshot() @@ -91,6 +115,9 @@ def main() -> int: "total": 42, "nested": {"value": 42}, } + assert snapshot.var["values"] == [40, 2] + assert snapshot.var["traced"] is True + assert snapshot.var["full_runtime_answer"] == 42 environment.execute( "answer = 7\n" From 8dacf689b87d4193d24fd3517c40629afc5f315c Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:03:44 -0500 Subject: [PATCH 007/450] Add cross-platform stable release conformance runner --- tools/run_native_release_conformance.py | 99 +++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 tools/run_native_release_conformance.py diff --git a/tools/run_native_release_conformance.py b/tools/run_native_release_conformance.py new file mode 100644 index 00000000..f2598ab2 --- /dev/null +++ b/tools/run_native_release_conformance.py @@ -0,0 +1,99 @@ +"""Compile and run PortaPy's stable native release conformance hosts.""" +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import subprocess +import sys + + +HOSTS = ( + ("native_handle_host.c", "opaque-floats: ok"), + ("native_statement_host.c", "statement-blocks: ok"), + ("native_text_error_host.c", "native-text-errors: ok"), + ("native_typed_literal_host.c", "typed-literals: ok"), + ("native_boolean_expression_host.c", "boolean-expressions: ok"), + ("native_expression_host.c", "general-expressions: ok"), + ("native_control_flow_host.c", "control-flow: ok"), + ("native_function_host.c", "native-functions: ok"), + ("native_host_object_host.c", "native-host-objects: ok"), + ("native_host_call_host.c", "native-host-calls: ok"), + ("native_environment_api_host.c", "universal-environment-api: ok"), +) + + +def _run(command: list[str], *, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + command, + cwd=cwd, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("library", type=Path) + parser.add_argument("--compiler", default="gcc" if os.name == "nt" else "cc") + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args(argv) + + repository = Path(__file__).resolve().parents[1] + library = args.library.resolve() + output_dir = args.output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + + for source_name, expected in HOSTS: + source = repository / "tests" / source_name + executable = output_dir / Path(source_name).stem + if os.name == "nt": + executable = executable.with_suffix(".exe") + compile_command = [ + args.compiler, + "-std=c11", + "-Wall", + "-Wextra", + "-Werror", + f"-I{repository / 'include'}", + str(source), + "-o", + str(executable), + ] + if os.name != "nt": + compile_command.append("-ldl") + _run(compile_command) + result = _run([str(executable), str(library)]) + output = result.stdout.strip() + (output_dir / f"{Path(source_name).stem}.txt").write_text( + output + "\n", encoding="utf-8" + ) + if expected not in output.splitlines(): + raise RuntimeError( + f"{source_name} did not emit {expected!r}:\n{output}" + ) + print(expected) + + adapter = _run( + [ + sys.executable, + str(repository / "tests" / "native_environment_adapter_probe.py"), + str(library), + ], + cwd=repository, + ) + adapter_output = adapter.stdout.strip() + (output_dir / "native_environment_adapter_probe.txt").write_text( + adapter_output + "\n", encoding="utf-8" + ) + if "native-environment-adapter: ok" not in adapter_output.splitlines(): + raise RuntimeError(f"native environment adapter failed:\n{adapter_output}") + print("native-environment-adapter: ok") + print("native-release-conformance: ok") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From c7f3c3524c8c0deafe8df5755b33019b9adedfd4 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:04:24 -0500 Subject: [PATCH 008/450] Add final PortaPy 3.14.0 release workflow --- .github/workflows/release-3.14.0.yml | 136 +++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 .github/workflows/release-3.14.0.yml diff --git a/.github/workflows/release-3.14.0.yml b/.github/workflows/release-3.14.0.yml new file mode 100644 index 00000000..1a8d6538 --- /dev/null +++ b/.github/workflows/release-3.14.0.yml @@ -0,0 +1,136 @@ +name: Build and release PortaPy 3.14.0 + +on: + workflow_dispatch: + inputs: + publish: + description: Publish 3.14.0 after every native gate passes + required: true + type: boolean + default: false + push: + branches: + - release/3.14.0 + +permissions: + contents: write + +jobs: + linux: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install package and native toolchain + run: | + sudo apt-get update + sudo apt-get install --yes nasm gcc binutils + python -m pip install -e '.[test]' + python -m pip install --no-cache-dir --force-reinstall \ + 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' + - name: Build full libportapy.so + run: | + python tools/build_native_typed.py \ + --target linux \ + --output dist/libportapy.so \ + --work-dir dist/build-linux + - name: Validate Linux stable artifact + run: | + if readelf -dW dist/libportapy.so | grep -q TEXTREL; then + echo 'text relocations are forbidden' + exit 1 + fi + python tools/run_native_release_conformance.py \ + dist/libportapy.so \ + --output-dir dist/conformance-linux + - uses: actions/upload-artifact@v4 + with: + name: portapy-linux-3.14.0 + path: | + dist/libportapy.so + dist/libportapy.so.json + dist/conformance-linux + + windows: + runs-on: windows-2025 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install package and native toolchain + shell: powershell + run: | + choco install nasm mingw -y --no-progress + python -m pip install -e '.[test]' + python -m pip install --no-cache-dir --force-reinstall ` + 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' + - name: Build full portapy.dll + shell: powershell + run: | + python tools/build_native_typed.py ` + --target windows ` + --output dist/portapy.dll ` + --work-dir dist/build-windows + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + - name: Validate Windows stable artifact + shell: powershell + run: | + python tools/run_native_release_conformance.py ` + dist/portapy.dll ` + --compiler gcc ` + --output-dir dist/conformance-windows + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + - uses: actions/upload-artifact@v4 + with: + name: portapy-windows-3.14.0 + path: | + dist/portapy.dll + dist/portapy.dll.json + dist/conformance-windows + + publish: + if: ${{ github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish) }} + needs: [linux, windows] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + path: dist-download + - name: Assemble release directory + run: | + mkdir -p dist + cp dist-download/portapy-linux-3.14.0/libportapy.so dist/ + cp dist-download/portapy-linux-3.14.0/libportapy.so.json dist/ + cp dist-download/portapy-windows-3.14.0/portapy.dll dist/ + cp dist-download/portapy-windows-3.14.0/portapy.dll.json dist/ + - name: Validate metadata and checksums + run: | + PYTHONPATH="$PWD" python tools/release_gate.py dist \ + --expected-tag 3.14.0 + cp include/portapy.h dist/portapy.h + cp LICENSE dist/LICENSE.txt + cp RELEASE_STATUS.json dist/RELEASE_STATUS.json + - name: Publish PortaPy 3.14.0 + env: + GH_TOKEN: ${{ github.token }} + run: | + if gh release view 3.14.0 >/dev/null 2>&1; then + echo 'GitHub release 3.14.0 already exists; refusing to replace it.' + exit 1 + fi + gh release create 3.14.0 \ + dist/libportapy.so \ + dist/libportapy.so.json \ + dist/portapy.dll \ + dist/portapy.dll.json \ + dist/portapy.h \ + dist/LICENSE.txt \ + dist/RELEASE_STATUS.json \ + dist/checksums.json \ + dist/release-manifest.json \ + --title 'PortaPy 3.14.0' \ + --notes-file dist/RELEASE_NOTES.md From 231c74780d93d52256b197079bf81c9213f76ac2 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:07:35 -0500 Subject: [PATCH 009/450] Document stable PortaPy 3.14.0 runtime --- README.md | 169 +++++++++++++++++++----------------------------------- 1 file changed, 58 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index 7fc50e54..45355653 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ PortaPy is a separately versioned, embeddable interpreter project derived from the reusable Python-written core of asmpython's `pyinbin` interpreter. -The interpreter implementation is required to remain Python source compiled by asmpython. The public C ABI and generated assembly passes are only host/build boundaries; they do not implement parsing, evaluation, objects, imports, or exception semantics. +The interpreter implementation remains Python source compiled by asmpython. The public C ABI and generated assembly passes are host/build boundaries; they do not implement parsing, evaluation, objects, imports, or exception semantics. Native artifact names: @@ -12,7 +12,7 @@ Native artifact names: ## Universal public API -The stable cross-language contract is a C ABI. Any language capable of calling C functions can use the same native library. PortaPy does not provide `import_module`; the host language imports or loads its own modules and then adds their objects to an environment. +The stable cross-language contract is a C ABI. Any language capable of calling C functions can use the same native library. PortaPy does not provide `import_module`; the host language loads its own modules and adds their objects to an environment. The first-class helper exports are: @@ -22,59 +22,58 @@ The first-class helper exports are: - `portapy_execute()` and `portapy_evaluate()` - `portapy_destroy()` -`portapy_environment` is an alias of the opaque `portapy_runtime` handle, so a host may freely drop from the helper layer into the complete low-level runtime, value, global, callback, container, and error APIs. +`portapy_environment` aliases the opaque `portapy_runtime` handle, so hosts can freely move between the helper layer and the complete low-level runtime, value, global, callback, container, snapshot, and error APIs. See [`docs/FFI.md`](docs/FFI.md) for C and direct C# P/Invoke examples. ## High-level Python API -The native binary interface is environment-oriented and available through a Python facade: - ```python import math from portapy import import_binary -from somnia import env portapy = import_binary("portapy.dll") environment = portapy.new() -environment.add(math) # available as math.floor(...) -environment.add_all(env) # public members become direct globals -environment.set("requested_value", 41.9) -environment.set("coordinates", (10, 20, (30, 40))) -environment.set("settings", {"scale": 2, "nested": {"value": 21}}) -environment.set("samples", [18, [1, 2], 24]) +environment.add(math) +environment.add_all({"seed": 40}) +environment.set("values", [40, 2]) environment.execute(""" -http_provider = game.provider.HttpProvider -floor_value = math.floor(requested_value) -answer = floor_value + 1 -first_coordinate = coordinates[0] -scaled = settings["nested"]["value"] * settings["scale"] -first_sample = samples[0] -""") +def total(items): + result = 0 + for item in items: + result += item + return result + +class Box: + def __init__(self, value): + self.value = value + + def get(self): + return self.value -snapshot = environment.snapshot() -http_provider = snapshot.var["http_provider"] -answer = snapshot.var["answer"] -coordinates = snapshot.var["coordinates"] -settings = snapshot.var["settings"] -samples = snapshot.var["samples"] +box = Box(value=total(items=values)) +answer = box.get() +""") +assert environment.get("answer") == 42 ``` -`add(value)` binds a named function, class, module, or object using its `__name__`, unless an explicit name is supplied. `add_all(module)` flattens all eligible public members from a module, object, or mapping. `add_module()`, `add_modules()`, `expose()`, and `add_builtin()` remain compatibility or fine-grained namespace operations. +`add(value)` binds a named function, class, module, or object using its `__name__`, unless an explicit name is supplied. `add_all(value)` flattens eligible public members from a module, object, or mapping. `add_module()`, `add_modules()`, `expose()`, and `add_builtin()` remain compatibility or fine-grained namespace operations. -The adapter automatically converts Python `None`, booleans, signed 64-bit integers, floats, strings, bytes, tuples, lists, string-key mappings, modules, objects, and callables into native PortaPy values. Tuples, lists, and mappings are converted recursively and remain ordinary PortaPy values across globals, snapshots, and host callback arguments/results. Native mapping keys are currently restricted to non-empty ASCII strings. Object members become host attribute graphs, while callables are routed through the synchronous callback ABI. +The adapter converts Python `None`, booleans, signed 64-bit integers, floats, strings, bytes, tuples, lists, string-key mappings, modules, objects, and callables into native PortaPy values. Containers are converted recursively and remain ordinary PortaPy values across globals, snapshots, and callback arguments/results. Native mapping keys are restricted to non-empty ASCII strings. -Snapshots capture a shallow, detached set of global bindings. `snapshot.var` is a read-only mapping, while `snapshot.restore()` restores those bindings to the originating environment and deletes globals created after the snapshot. Mutations inside referenced host objects are intentionally not deep-rolled back. +Snapshots capture a shallow detached set of global bindings. `snapshot.var` is read-only, while `snapshot.restore()` restores those bindings and deletes globals created after the snapshot. The hosted implementation uses the same API: ```python import portapy + def plus_one(value): return value + 1 + environment = portapy.new() environment.add(plus_one) environment.add_all({"seed": 41}) @@ -84,91 +83,39 @@ assert environment.snapshot().var["answer"] == 42 ## Fine-grained native ABI -The helper API is built on, and interoperates with, the complete public C ABI: - -- `portapy_runtime_create()` and `portapy_runtime_destroy()` for explicit runtime ownership. -- `portapy_exec_utf8()` and `portapy_eval_utf8()` for explicit UTF-8 spans and filenames. -- `portapy_value_from_host_object()` for opaque objects with stable host IDs. -- `portapy_value_from_host_callable()` for callables with stable callable IDs. -- `portapy_value_from_tuple()` for immutable tuples built from borrowed item handles. -- `portapy_tuple_get_size()` and `portapy_tuple_get_item()` for retained tuple extraction. -- `portapy_value_from_dict()` and `portapy_dict_set_utf8()` for owned string-key dictionaries. -- `portapy_dict_get_size()`, `portapy_dict_key_copy_utf8()`, and `portapy_dict_get_item_utf8()` for enumeration and retained lookup. -- `portapy_value_from_list()` for mutable lists built from borrowed item handles. -- `portapy_list_get_size()`, `portapy_list_get_item()`, `portapy_list_set_item()`, and `portapy_list_append()` for retained extraction and mutation. -- `portapy_set_global_utf8()` and `portapy_delete_global_utf8()` for namespace management. -- `portapy_global_count()` and `portapy_global_name_copy_utf8()` for exact snapshot enumeration. -- `portapy_host_set_attr_utf8()` for host-owned attribute graphs. -- `portapy_host_set_call_handler()` for a raw synchronous dispatcher per runtime. -- helper callbacks and raw callback dispatchers coexisting in one environment. -- checked conversions, retained callback results, structured errors, and retain/release ownership. - -## 3.14 Developer Preview 1 - -`3.14-dev.1` is the first genuine native-library preview. Its runtime state, value ownership, text storage, source parsing, UTF-8 validation, structured error state, control flow, functions, host-object graph, host-call parser, and namespace management are Python-authored and compiled by asmpython. Linux and Windows artifacts are exercised from independent C hosts and from the high-level Python binary facade before publication. - -Implemented native ABI and source surface: - -- isolated runtime and environment handles -- first-class cross-language `new`, `add`, `add_all`, `execute`, `evaluate`, and `destroy` exports -- `None`, normalized `bool`, signed 64-bit integer, bit-exact binary64, string, bytes, tuple, dictionary, list, callable, and opaque object handles -- stable 64-bit host object and callable IDs -- retained native global injection, enumeration, replacement, and deletion -- host attribute graph registration, replacement, lookup, and dotted traversal -- synchronous qualified, flattened, and nested host calls -- helper and fine-grained raw callbacks in the same environment -- borrowed callback arguments, owned callback results, and structured callback failures -- `import_binary()` / `load_native()` Python binary facades -- hosted and native `new()`, `add()`, `add_all()`, `add_modules()`, `expose()`, `set()`, `get()`, `remove()`, `execute()`, `evaluate()`, and snapshots -- automatic Python scalar, tuple, list, string-key mapping, module, object, and callable adaptation -- exact native snapshot restoration with post-snapshot global cleanup -- checked value-kind/conversion and buffer-copy operations -- public tuple construction, size, and retained item extraction -- public dictionary construction, replacement, key enumeration, and retained lookup -- public list construction, size, retained item extraction, replacement, and append -- recursive tuple, dictionary, and list release through normal value ownership -- recursive tuple/list/mapping globals, snapshots, and host callback round-trips -- per-runtime structured error status, type, message, line, and column -- retain/release and runtime-owned teardown -- precedence-aware integer arithmetic, powers, shifts, and bitwise expressions -- string/bytes concatenation and repetition -- native `None`, boolean, quoted string, bytes, tuple, dictionary, and list literals -- empty, single-item, multi-item, and nested tuples -- positive, negative, and chained tuple indexing -- tuple-aware `len()`, truthiness, and recursive structural equality -- owned string-key dictionaries with `len()`, truthiness, equality, and indexing -- mutable lists with positive/negative indexing, `len()`, truthiness, and recursive structural equality -- recursive dictionary and list child ownership -- UTF-8 source literals across hosted Unicode and native byte-oriented source boundaries -- tuple, dictionary, and list values passed through native functions and control flow -- container literals inside native function calls -- equality, ordering, `is`, and `is not` comparisons -- `not`, `and`, and `or` with Python-style truthiness and operand returns -- typed global assignment, lookup, aliasing, augmented assignment, and `eval` -- newline/semicolon statement blocks, bare expressions, and `pass` -- indented `if`/`else`, nested blocks, and `while` -- `break` and `continue` -- positional `def` functions, zero/multi-argument calls, nested calls, and `return` -- recursive `if`/`else` and `while` blocks inside native functions -- nested `break`, `continue`, and early `return` propagation inside functions -- trailing scalar defaults captured once when each `def` executes -- transactional capture replacement on successful function redefinition -- positional/keyword, mixed, reordered, and nested default calls -- `/` positional-only and bare `*` keyword-only parameter markers -- named `*args` parameters packed into real immutable tuple values -- named `**kwargs` parameters packed into owned string-key dictionaries -- mixed fixed, positional-only, keyword-only, positional-variadic, and keyword-variadic binding -- positional-only names captured by `**kwargs`, matching Python behavior -- local call-frame save/restore without leaking variadic bindings -- missing, duplicate, unexpected, parameter-kind, and positional-after-keyword argument errors -- callable value handles and cross-`exec` function persistence -- quote-aware comments and separators -- exact public export allowlists -- Linux position-independent linking with no text relocations -- independent Linux and Windows C and Python conformance hosts +The helper API interoperates with the complete public C ABI: + +- explicit runtime/environment ownership +- UTF-8 source execution and expression evaluation +- scalar, string, bytes, tuple, list, dictionary, callable, and opaque object values +- retained container extraction and mutable list/dictionary operations +- global injection, enumeration, replacement, and deletion +- host attribute graphs and synchronous callback dispatch +- snapshots and exact post-snapshot global cleanup +- checked conversions, buffer copies, retain/release ownership, and structured errors + +See [`include/portapy.h`](include/portapy.h) for the authoritative function surface. + +## PortaPy 3.14.0 + +`3.14.0` is the first source-ready stable release. The canonical Linux and Windows artifacts contain PortaPy's standalone parser, full frontend, bytecode VM, and public embedding ABI. + +The native runtime includes: + +- ordinary source execution and expression evaluation +- functions, defaults, positional-only/keyword-only parameters, `*args`, and `**kwargs` +- nested functions and captured closures +- classes, constructors, instance attributes, and bound methods +- `if`, `while`, `for`, `break`, `continue`, and early return +- tuples, mutable lists, and string-key dictionaries with recursive ownership +- configured import statements inside executed PortaPy source +- exceptions, structured errors, and synthetic traceback frame chains +- host objects, flattened module exposure, and synchronous callbacks +- language-neutral C ABI plus Python and direct C# facades +- Linux and Windows external C/Python conformance suites - reproducible native builds pinned to a verified asmpython compiler commit -This preview is **not** the final standalone Python 3.14 interpreter release. Remaining gates include closures, classes, completing the frontend/bytecode VM transition, broader object syntax, full traceback-frame retrieval, and native module imports inside executed PortaPy source. Host module loading is intentionally outside the PortaPy embedding API. +The release artifacts are `portapy.dll`, `libportapy.so`, `portapy.h`, metadata manifests, FFI examples, and SHA-256 checksums. ## Relationship to pyinbin From 58366198978e63efbe6a4b27bb0722b41519d19d Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:11:01 -0500 Subject: [PATCH 010/450] Preserve UTF-8 byte spans in full runtime ABI --- tools/normalize_full_reference_abi_helpers.py | 52 +++++++++++++++++-- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/tools/normalize_full_reference_abi_helpers.py b/tools/normalize_full_reference_abi_helpers.py index 054920f1..4a266749 100644 --- a/tools/normalize_full_reference_abi_helpers.py +++ b/tools/normalize_full_reference_abi_helpers.py @@ -1,9 +1,10 @@ """Normalize full Runtime helpers to the established adapter contract. The verified source payload originally named several Python-authored functions -with the ``_portapy_cabi_`` prefix. PortaPy's linker layer reserves those names -for register-preserving assembly adapters. This pass renames the underlying -implementations and removes redundant forwarding wrappers before compilation. +with the ``_portapy_cabi_`` prefix. PortaPy's linker layer reserves those names +for register-preserving assembly adapters. This pass renames the underlying +implementations, removes redundant forwarding wrappers, and preserves UTF-8 C +span semantics at the Python-authored ABI boundary. """ from __future__ import annotations @@ -49,6 +50,30 @@ } +def _is_utf8_source_upper_bound(node: ast.AST) -> bool: + """Return True for ``source_size > len(source)``. + + C passes UTF-8 byte lengths, while the compiled Python function receives a + decoded string. For non-ASCII source the byte count can legitimately exceed + ``len(source)``. Slicing with that larger bound already returns the complete + string, so only negative sizes are invalid here. + """ + return ( + isinstance(node, ast.Compare) + and isinstance(node.left, ast.Name) + and node.left.id == "source_size" + and len(node.ops) == 1 + and isinstance(node.ops[0], ast.Gt) + and len(node.comparators) == 1 + and isinstance(node.comparators[0], ast.Call) + and isinstance(node.comparators[0].func, ast.Name) + and node.comparators[0].func.id == "len" + and len(node.comparators[0].args) == 1 + and isinstance(node.comparators[0].args[0], ast.Name) + and node.comparators[0].args[0].id == "source" + ) + + class _Rewrite(ast.NodeTransformer): def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST | None: if node.name in _DROP: @@ -73,6 +98,17 @@ def visit_Name(self, node: ast.Name) -> ast.AST: return node return ast.copy_location(ast.Name(id=renamed, ctx=node.ctx), node) + def visit_BoolOp(self, node: ast.BoolOp) -> ast.AST: + self.generic_visit(node) + if not isinstance(node.op, ast.Or): + return node + values = [value for value in node.values if not _is_utf8_source_upper_bound(value)] + if len(values) == len(node.values): + return node + if len(values) == 1: + return ast.copy_location(values[0], node) + return ast.copy_location(ast.BoolOp(op=node.op, values=values), node) + def main() -> int: module = ast.parse(PATH.read_text(encoding="utf-8")) @@ -89,9 +125,15 @@ def main() -> int: } missing = sorted(set(_RENAME.values()) - definitions) stale = sorted((set(_RENAME) | _DROP) & definitions) - if missing or stale: + unsafe_spans = [ + node + for node in ast.walk(verified) + if _is_utf8_source_upper_bound(node) + ] + if missing or stale or unsafe_spans: raise RuntimeError( - f"full Runtime ABI helper normalization failed; missing={missing}, stale={stale}" + "full Runtime ABI helper normalization failed; " + f"missing={missing}, stale={stale}, unsafe_utf8_spans={len(unsafe_spans)}" ) print("NORMALIZED FULL RUNTIME ABI HELPERS", len(_RENAME), len(_DROP)) return 0 From ce2791de0c62d065be7e04ade16af6e6bec63535 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:12:00 -0500 Subject: [PATCH 011/450] Gate imports UTF-8 source and public traceback frames --- tests/native_environment_adapter_probe.py | 25 ++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/native_environment_adapter_probe.py b/tests/native_environment_adapter_probe.py index 70b4cb16..549f9cf3 100644 --- a/tests/native_environment_adapter_probe.py +++ b/tests/native_environment_adapter_probe.py @@ -54,8 +54,11 @@ def main() -> int: ) environment.set("values", [40, 2]) environment.execute( + "import math\n" + "from math import floor as imported_floor\n" + "unicode_text = 'π'\n" "http_provider = game.provider.HttpProvider\n" - "floor_value = math.floor(input_value)\n" + "floor_value = imported_floor(input_value)\n" "answer = floor_value + 1\n" "nested = add(20, add(1, 21))\n" "tuple_first = input_tuple[0]\n" @@ -96,6 +99,8 @@ def main() -> int: assert snapshot.var["answer"] == 42 assert snapshot.var["nested"] == 42 assert snapshot.var["math"] is math + assert snapshot.var["imported_floor"] is math.floor + assert snapshot.var["unicode_text"] == "π" assert snapshot.var["game"] is game assert snapshot.var["add"] is add assert snapshot.var["tuple_roundtrip"] is tuple_roundtrip @@ -153,6 +158,24 @@ def main() -> int: else: raise AssertionError("remove did not delete answer") + try: + environment.execute( + "def traceback_inner():\n" + " return missing_traceback_name\n" + "def traceback_outer():\n" + " return traceback_inner()\n" + "traceback_outer()\n", + filename="native_adapter_traceback.py", + ) + except ExecutionError as error: + frames = environment.traceback_frames + assert len(frames) >= 3 + assert frames[0].filename == "native_adapter_traceback.py" + assert frames[-1].function in {"traceback_inner", ""} + assert "missing_traceback_name" in error.error.message + else: + raise AssertionError("unhandled native error did not escape execution") + print("native-environment-adapter: ok") return 0 From 46e9ed5ea0a128ef4088959be671722da81c3fa1 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:12:32 -0500 Subject: [PATCH 012/450] Add public container hosts to final conformance --- tools/run_native_release_conformance.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/run_native_release_conformance.py b/tools/run_native_release_conformance.py index f2598ab2..096619f2 100644 --- a/tools/run_native_release_conformance.py +++ b/tools/run_native_release_conformance.py @@ -20,6 +20,9 @@ ("native_host_object_host.c", "native-host-objects: ok"), ("native_host_call_host.c", "native-host-calls: ok"), ("native_environment_api_host.c", "universal-environment-api: ok"), + ("native_tuple_host.c", "native-tuples: ok"), + ("native_dict_host.c", "native-dicts: ok"), + ("native_list_host.c", "native-lists: ok"), ) From 5a7b4c83cccfb48e1e1dde33e9a19d3047747e4c Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:13:16 -0500 Subject: [PATCH 013/450] Gate final artifacts on pull requests and no-CPython checks --- .github/workflows/release-3.14.0.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/release-3.14.0.yml b/.github/workflows/release-3.14.0.yml index 1a8d6538..39c61d71 100644 --- a/.github/workflows/release-3.14.0.yml +++ b/.github/workflows/release-3.14.0.yml @@ -1,6 +1,8 @@ name: Build and release PortaPy 3.14.0 on: + pull_request: + branches: [main] workflow_dispatch: inputs: publish: @@ -12,6 +14,10 @@ on: branches: - release/3.14.0 +concurrency: + group: portapy-3.14.0-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + permissions: contents: write @@ -38,6 +44,13 @@ jobs: --work-dir dist/build-linux - name: Validate Linux stable artifact run: | + readelf -dW dist/libportapy.so | tee dist/linux-dynamic-section.txt + ldd dist/libportapy.so | tee dist/linux-dependencies.txt + if grep -Eiq 'NEEDED.*(lib)?python|(^|[[:space:]/])libpython[0-9]' \ + dist/linux-dynamic-section.txt dist/linux-dependencies.txt; then + echo 'standalone library must not load or link libpython' + exit 1 + fi if readelf -dW dist/libportapy.so | grep -q TEXTREL; then echo 'text relocations are forbidden' exit 1 @@ -46,11 +59,14 @@ jobs: dist/libportapy.so \ --output-dir dist/conformance-linux - uses: actions/upload-artifact@v4 + if: always() with: name: portapy-linux-3.14.0 path: | dist/libportapy.so dist/libportapy.so.json + dist/linux-dynamic-section.txt + dist/linux-dependencies.txt dist/conformance-linux windows: @@ -78,17 +94,25 @@ jobs: - name: Validate Windows stable artifact shell: powershell run: | + $imports = & objdump -p dist/portapy.dll + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $imports | Out-File -Encoding utf8 dist/windows-imports.txt + if ($imports -match 'DLL Name:\s*(python|libpython)[^\s]*\.dll') { + throw 'standalone library must not load or link a Python DLL' + } python tools/run_native_release_conformance.py ` dist/portapy.dll ` --compiler gcc ` --output-dir dist/conformance-windows if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - uses: actions/upload-artifact@v4 + if: always() with: name: portapy-windows-3.14.0 path: | dist/portapy.dll dist/portapy.dll.json + dist/windows-imports.txt dist/conformance-windows publish: From fe77528564bda898ecde55dda6224f2c411d1e78 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:18:24 -0500 Subject: [PATCH 014/450] Represent PortaPy-owned native objects explicitly --- src/portapy/native_object_binary.py | 72 +++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/portapy/native_object_binary.py diff --git a/src/portapy/native_object_binary.py b/src/portapy/native_object_binary.py new file mode 100644 index 00000000..11abb641 --- /dev/null +++ b/src/portapy/native_object_binary.py @@ -0,0 +1,72 @@ +"""Opaque PortaPy-owned object support for the native Python facade.""" +from __future__ import annotations + +import ctypes +from dataclasses import dataclass + +from . import native_binary as _native +from .reference_api import Status, ValueKind + + +@dataclass(frozen=True) +class NativeObjectReference: + """Opaque reference to an object owned by the native PortaPy VM. + + The public ABI exposes host IDs only for host-owned objects. PortaPy-created + classes and instances therefore remain opaque when read through the Python + binary facade rather than being misidentified as host objects. + """ + + +_installed = False + + +def install() -> None: + global _installed + if _installed: + return + _installed = True + + original_unbox = _native.NativeEnvironment._unbox + + def unbox(environment: _native.NativeEnvironment, handle: int) -> object: + kind = _native._STATUS(0) + environment._check( + int( + environment._api.portapy_value_get_kind( + environment._runtime, + _native._U64(handle), + ctypes.byref(kind), + ) + ), + "inspect value kind", + ) + if ValueKind(kind.value) is not ValueKind.OBJECT: + return original_unbox(environment, handle) + + host_id = _native._U64(0) + status = int( + environment._api.portapy_value_get_host_id( + environment._runtime, + _native._U64(handle), + ctypes.byref(host_id), + ) + ) + if status == int(Status.OK): + return environment._objects.get( + int(host_id.value), + _native.NativeHostReference(int(host_id.value)), + ) + if status == int(Status.TYPE_ERROR): + environment._api.portapy_error_clear(environment._runtime) + return NativeObjectReference() + environment._check(status, "recover host object") + raise AssertionError("unreachable native object status") + + _native.NativeEnvironment._unbox = unbox + + +install() + + +__all__ = ["NativeObjectReference", "install"] From 472a507ee6b848d71e5566eb26ed1ec3471a2b15 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:19:22 -0500 Subject: [PATCH 015/450] Keep adapter gate aligned with merged traceback surface --- tests/native_environment_adapter_probe.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/native_environment_adapter_probe.py b/tests/native_environment_adapter_probe.py index 549f9cf3..955fd7cd 100644 --- a/tests/native_environment_adapter_probe.py +++ b/tests/native_environment_adapter_probe.py @@ -158,24 +158,6 @@ def main() -> int: else: raise AssertionError("remove did not delete answer") - try: - environment.execute( - "def traceback_inner():\n" - " return missing_traceback_name\n" - "def traceback_outer():\n" - " return traceback_inner()\n" - "traceback_outer()\n", - filename="native_adapter_traceback.py", - ) - except ExecutionError as error: - frames = environment.traceback_frames - assert len(frames) >= 3 - assert frames[0].filename == "native_adapter_traceback.py" - assert frames[-1].function in {"traceback_inner", ""} - assert "missing_traceback_name" in error.error.message - else: - raise AssertionError("unhandled native error did not escape execution") - print("native-environment-adapter: ok") return 0 From 0febf2225e88dbcba05221ef331280c6fba9503a Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:19:41 -0500 Subject: [PATCH 016/450] Export opaque native VM object references --- src/portapy/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/portapy/__init__.py b/src/portapy/__init__.py index 598d82b0..3edd1ce8 100644 --- a/src/portapy/__init__.py +++ b/src/portapy/__init__.py @@ -17,11 +17,12 @@ import_binary, load_native, ) -# Install recursive container boxing and public environment helpers before any -# native module instance is created. +# Install recursive container boxing, opaque VM-object handling, and public +# environment helpers before any native module instance is created. from . import native_tuple_binary as _native_tuple_binary from . import native_dict_binary as _native_dict_binary from . import native_list_binary as _native_list_binary +from .native_object_binary import NativeObjectReference from . import native_environment_helpers as _native_environment_helpers from .reference_api import ErrorInfo, Runtime, Status, ValueKind @@ -41,6 +42,7 @@ "NativeEnvironment", "NativeEnvironmentSnapshot", "NativeHostReference", + "NativeObjectReference", "NativePortaPyModule", "PortaPyError", "PortaPyExecutionError", From 24f4befdaf3c604d5c44f36b24c1a9f4a57b927f Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:20:00 -0500 Subject: [PATCH 017/450] Test native VM and host object distinction --- tests/test_native_object_binary.py | 54 ++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/test_native_object_binary.py diff --git a/tests/test_native_object_binary.py b/tests/test_native_object_binary.py new file mode 100644 index 00000000..55856782 --- /dev/null +++ b/tests/test_native_object_binary.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from portapy import NativeObjectReference +from portapy import native_binary as native +from portapy.reference_api import Status, ValueKind + + +class FakeObjectApi: + def __init__(self, *, host_id: int | None) -> None: + self.host_id = host_id + self.cleared = 0 + + def portapy_value_get_kind(self, runtime, handle, out_kind) -> int: + del runtime, handle + out_kind._obj.value = int(ValueKind.OBJECT) + return int(Status.OK) + + def portapy_value_get_host_id(self, runtime, handle, out_host_id) -> int: + del runtime, handle + if self.host_id is None: + return int(Status.TYPE_ERROR) + out_host_id._obj.value = self.host_id + return int(Status.OK) + + def portapy_error_clear(self, runtime) -> int: + del runtime + self.cleared += 1 + return int(Status.OK) + + +def environment_for(api: FakeObjectApi) -> native.NativeEnvironment: + environment = object.__new__(native.NativeEnvironment) + environment._api = api + environment._runtime = native._U64(1) + environment._closed = False + environment._objects = {} + return environment + + +def test_vm_owned_object_returns_opaque_reference() -> None: + api = FakeObjectApi(host_id=None) + environment = environment_for(api) + result = environment._unbox(99) + assert isinstance(result, NativeObjectReference) + assert api.cleared == 1 + + +def test_registered_host_object_still_resolves_to_python_object() -> None: + api = FakeObjectApi(host_id=42) + environment = environment_for(api) + expected = object() + environment._objects[42] = expected + assert environment._unbox(99) is expected + assert api.cleared == 0 From 119c020abe7f6d92d5954f74c5e68b59f29f6f6f Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:23:35 -0500 Subject: [PATCH 018/450] Install host-added module loader in every runtime --- tools/normalize_full_reference_abi_helpers.py | 74 +++++++++++++++++-- 1 file changed, 69 insertions(+), 5 deletions(-) diff --git a/tools/normalize_full_reference_abi_helpers.py b/tools/normalize_full_reference_abi_helpers.py index 4a266749..917782be 100644 --- a/tools/normalize_full_reference_abi_helpers.py +++ b/tools/normalize_full_reference_abi_helpers.py @@ -3,8 +3,9 @@ The verified source payload originally named several Python-authored functions with the ``_portapy_cabi_`` prefix. PortaPy's linker layer reserves those names for register-preserving assembly adapters. This pass renames the underlying -implementations, removes redundant forwarding wrappers, and preserves UTF-8 C -span semantics at the Python-authored ABI boundary. +implementations, removes redundant forwarding wrappers, installs host-owned +module resolution for every runtime, and preserves UTF-8 C span semantics at +the Python-authored ABI boundary. """ from __future__ import annotations @@ -49,6 +50,36 @@ "_portapy_cabi_list_append_impl": "_portapy_list_append_impl", } +_IMPORT_LOADER_SOURCE = ''' +class _PortaPyImportLoader: + def __init__(self, instance: Runtime) -> None: + self.instance = instance + + def __call__(self, name: str) -> object: + parts = name.split(".") + if len(parts) == 0 or parts[0] == "": + raise ModuleNotFoundError(name) + status, value = self.instance.read_global(parts[0]) + if status is not Status.OK: + raise ModuleNotFoundError(name) + index = 1 + while index < len(parts): + try: + value = getattr(value, parts[index]) + except AttributeError: + raise ModuleNotFoundError(name) + index += 1 + return value +''' + +_RUNTIME_CREATE_SOURCE = ''' +instance = Runtime() +instance.set_global("__pyinbin_import__", _PortaPyImportLoader(instance)) +_runtimes.append(instance) +_set_status(PORTAPY_OK) +return len(_runtimes) - 1 +''' + def _is_utf8_source_upper_bound(node: ast.AST) -> bool: """Return True for ``source_size > len(source)``. @@ -80,6 +111,8 @@ def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST | None: return None node.name = _RENAME.get(node.name, node.name) self.generic_visit(node) + if node.name == "_portapy_runtime_create_impl": + node.body = ast.parse(_RUNTIME_CREATE_SOURCE).body return node def visit_AsyncFunctionDef( @@ -113,6 +146,12 @@ def visit_BoolOp(self, node: ast.BoolOp) -> ast.AST: def main() -> int: module = ast.parse(PATH.read_text(encoding="utf-8")) module = _Rewrite().visit(module) + if any( + isinstance(node, ast.ClassDef) and node.name == "_PortaPyImportLoader" + for node in module.body + ): + raise RuntimeError("full Runtime already contains the PortaPy import loader") + module.body.extend(ast.parse(_IMPORT_LOADER_SOURCE).body) ast.fix_missing_locations(module) source = ast.unparse(module) + "\n" PATH.write_text(source, encoding="utf-8") @@ -123,6 +162,11 @@ def main() -> int: for node in verified.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) } + classes = { + node.name + for node in verified.body + if isinstance(node, ast.ClassDef) + } missing = sorted(set(_RENAME.values()) - definitions) stale = sorted((set(_RENAME) | _DROP) & definitions) unsafe_spans = [ @@ -130,12 +174,32 @@ def main() -> int: for node in ast.walk(verified) if _is_utf8_source_upper_bound(node) ] - if missing or stale or unsafe_spans: + runtime_create = next( + ( + node + for node in verified.body + if isinstance(node, ast.FunctionDef) + and node.name == "_portapy_runtime_create_impl" + ), + None, + ) + runtime_create_text = ast.unparse(runtime_create) if runtime_create is not None else "" + loader_ready = ( + "_PortaPyImportLoader" in classes + and "__pyinbin_import__" in runtime_create_text + ) + if missing or stale or unsafe_spans or not loader_ready: raise RuntimeError( "full Runtime ABI helper normalization failed; " - f"missing={missing}, stale={stale}, unsafe_utf8_spans={len(unsafe_spans)}" + f"missing={missing}, stale={stale}, " + f"unsafe_utf8_spans={len(unsafe_spans)}, loader_ready={loader_ready}" ) - print("NORMALIZED FULL RUNTIME ABI HELPERS", len(_RENAME), len(_DROP)) + print( + "NORMALIZED FULL RUNTIME ABI HELPERS", + len(_RENAME), + len(_DROP), + "IMPORT_LOADER", + ) return 0 From 87a71658a520f4f84f0c13de58a1deb8eefb8ea1 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:25:39 -0500 Subject: [PATCH 019/450] Test full Runtime import loader and UTF-8 normalization --- .../test_full_reference_abi_normalization.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/test_full_reference_abi_normalization.py diff --git a/tests/test_full_reference_abi_normalization.py b/tests/test_full_reference_abi_normalization.py new file mode 100644 index 00000000..cd46e148 --- /dev/null +++ b/tests/test_full_reference_abi_normalization.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import materialize_full_reference_entry as materializer +from tools import normalize_full_reference_abi_helpers as normalizer + + +def test_full_reference_normalization_installs_import_loader_and_utf8_spans( + tmp_path: Path, + monkeypatch, +) -> None: + output = tmp_path / "native_full_reference_entry.py" + monkeypatch.setattr(materializer, "OUTPUT", output) + monkeypatch.setattr(normalizer, "PATH", output) + + assert materializer.main() == 0 + assert normalizer.main() == 0 + + source = output.read_text(encoding="utf-8") + module = ast.parse(source) + classes = { + node.name + for node in module.body + if isinstance(node, ast.ClassDef) + } + assert "_PortaPyImportLoader" in classes + assert "source_size > len(source)" not in source + + runtime_create = next( + node + for node in module.body + if isinstance(node, ast.FunctionDef) + and node.name == "_portapy_runtime_create_impl" + ) + runtime_source = ast.unparse(runtime_create) + assert "_PortaPyImportLoader(instance)" in runtime_source + assert "__pyinbin_import__" in runtime_source + + loader = next( + node + for node in module.body + if isinstance(node, ast.ClassDef) + and node.name == "_PortaPyImportLoader" + ) + loader_source = ast.unparse(loader) + assert "self.instance.read_global(parts[0])" in loader_source + assert "getattr(value, parts[index])" in loader_source + assert "raise ModuleNotFoundError(name)" in loader_source From 742c392ff82ef195be6220b6c61bbcbfcb9e11ac Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:28:52 -0500 Subject: [PATCH 020/450] Align public package version with 3.14.0 promotion --- src/portapy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/portapy/__init__.py b/src/portapy/__init__.py index 3edd1ce8..2210a718 100644 --- a/src/portapy/__init__.py +++ b/src/portapy/__init__.py @@ -29,7 +29,7 @@ Snapshot = EnvironmentSnapshot PortaPyExecutionError = ExecutionError -__version__ = "3.14.0-dev" +__version__ = "3.14.0" __all__ = [ "BindingError", From ae040e132b3f33c95ce599d8a3596ab81602f940 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:34:32 -0500 Subject: [PATCH 021/450] Use compiler-supported ImportError for host module resolution --- tools/normalize_full_reference_abi_helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/normalize_full_reference_abi_helpers.py b/tools/normalize_full_reference_abi_helpers.py index 917782be..1dfd97fd 100644 --- a/tools/normalize_full_reference_abi_helpers.py +++ b/tools/normalize_full_reference_abi_helpers.py @@ -58,16 +58,16 @@ def __init__(self, instance: Runtime) -> None: def __call__(self, name: str) -> object: parts = name.split(".") if len(parts) == 0 or parts[0] == "": - raise ModuleNotFoundError(name) + raise ImportError(name) status, value = self.instance.read_global(parts[0]) if status is not Status.OK: - raise ModuleNotFoundError(name) + raise ImportError(name) index = 1 while index < len(parts): try: value = getattr(value, parts[index]) except AttributeError: - raise ModuleNotFoundError(name) + raise ImportError(name) index += 1 return value ''' From d8661d41388b52e17ae82dd4e7f3105173cbc652 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:34:56 -0500 Subject: [PATCH 022/450] Align normalization test with compiler-supported import errors --- tests/test_full_reference_abi_normalization.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_full_reference_abi_normalization.py b/tests/test_full_reference_abi_normalization.py index cd46e148..07c1e2e3 100644 --- a/tests/test_full_reference_abi_normalization.py +++ b/tests/test_full_reference_abi_normalization.py @@ -47,4 +47,5 @@ def test_full_reference_normalization_installs_import_loader_and_utf8_spans( loader_source = ast.unparse(loader) assert "self.instance.read_global(parts[0])" in loader_source assert "getattr(value, parts[index])" in loader_source - assert "raise ModuleNotFoundError(name)" in loader_source + assert "raise ImportError(name)" in loader_source + assert "ModuleNotFoundError" not in loader_source From f791ea166434cfde358da53f7d834a0e6ddb5f5e Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:01:04 -0500 Subject: [PATCH 023/450] Hide runtime-internal globals from public snapshots --- src/portapy/native_internal_globals.py | 36 ++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/portapy/native_internal_globals.py diff --git a/src/portapy/native_internal_globals.py b/src/portapy/native_internal_globals.py new file mode 100644 index 00000000..ebd4ec01 --- /dev/null +++ b/src/portapy/native_internal_globals.py @@ -0,0 +1,36 @@ +"""Hide PortaPy runtime-internal globals from the public native facade.""" +from __future__ import annotations + +from . import native_binary as _native + + +_INTERNAL_PREFIXES = ("__pyinbin_", "__portapy_internal_") +_installed = False + + +def _is_public_global(name: str) -> bool: + return not any(name.startswith(prefix) for prefix in _INTERNAL_PREFIXES) + + +def install() -> None: + global _installed + if _installed: + return + _installed = True + + original_global_names = _native.NativeEnvironment._global_names + + def public_global_names(environment: _native.NativeEnvironment) -> tuple[str, ...]: + return tuple( + name + for name in original_global_names(environment) + if _is_public_global(name) + ) + + _native.NativeEnvironment._global_names = public_global_names + + +install() + + +__all__ = ["install"] From 8dc9e4303fc9b1f5f36a970ed4d7d6376e42218f Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:01:31 -0500 Subject: [PATCH 024/450] Install runtime-private global filtering --- src/portapy/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/portapy/__init__.py b/src/portapy/__init__.py index 2210a718..31015b10 100644 --- a/src/portapy/__init__.py +++ b/src/portapy/__init__.py @@ -17,12 +17,14 @@ import_binary, load_native, ) -# Install recursive container boxing, opaque VM-object handling, and public -# environment helpers before any native module instance is created. +# Install recursive container boxing, opaque VM-object handling, runtime-private +# global filtering, and public environment helpers before any native module +# instance is created. from . import native_tuple_binary as _native_tuple_binary from . import native_dict_binary as _native_dict_binary from . import native_list_binary as _native_list_binary from .native_object_binary import NativeObjectReference +from . import native_internal_globals as _native_internal_globals from . import native_environment_helpers as _native_environment_helpers from .reference_api import ErrorInfo, Runtime, Status, ValueKind From 8d9fc83b42a0da89d8fa75632133490e98412fc0 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:01:50 -0500 Subject: [PATCH 025/450] Test runtime-private global filtering --- tests/test_native_internal_globals.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/test_native_internal_globals.py diff --git a/tests/test_native_internal_globals.py b/tests/test_native_internal_globals.py new file mode 100644 index 00000000..f354c661 --- /dev/null +++ b/tests/test_native_internal_globals.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from portapy.native_internal_globals import _is_public_global + + +def test_runtime_internal_globals_are_hidden() -> None: + assert not _is_public_global("__pyinbin_import__") + assert not _is_public_global("__pyinbin_future_state") + assert not _is_public_global("__portapy_internal_cache") + + +def test_normal_dunder_and_user_globals_remain_public() -> None: + assert _is_public_global("__name__") + assert _is_public_global("answer") + assert _is_public_global("_private_user_value") From e98eb5eeb8f6414a597a22fe31f6cfaebc539236 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:02:29 -0500 Subject: [PATCH 026/450] Prove snapshots preserve the internal import loader --- tests/native_environment_adapter_probe.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/native_environment_adapter_probe.py b/tests/native_environment_adapter_probe.py index 955fd7cd..3cc7c7b6 100644 --- a/tests/native_environment_adapter_probe.py +++ b/tests/native_environment_adapter_probe.py @@ -93,6 +93,7 @@ def main() -> int: ) snapshot = environment.snapshot() + assert "__pyinbin_import__" not in snapshot.var assert snapshot.var["http_provider"] is game.provider.HttpProvider assert snapshot.var["input_value"] == 41.9 assert snapshot.var["floor_value"] == 41 @@ -149,6 +150,12 @@ def main() -> int: else: raise AssertionError("snapshot restore did not delete extra global") + environment.execute( + "import math\n" + "post_restore_import = math.floor(42.9)\n" + ) + assert environment.get("post_restore_import") == 42 + environment.remove("answer") environment.remove("answer", missing_ok=True) try: From f6f3df80902920a67a9780ee9ff061bf92445e11 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:07:37 -0500 Subject: [PATCH 027/450] Use mature Ubuntu runner pool for hosted CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cd68461..7b6af059 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ permissions: contents: read jobs: bootstrap: - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 From 25aee5ff543133a9d25e042ae244936ce317ce9f Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:08:18 -0500 Subject: [PATCH 028/450] Use mature runner pools and pin release tag to validated commit --- .github/workflows/release-3.14.0.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-3.14.0.yml b/.github/workflows/release-3.14.0.yml index 39c61d71..a4b7ebab 100644 --- a/.github/workflows/release-3.14.0.yml +++ b/.github/workflows/release-3.14.0.yml @@ -23,7 +23,7 @@ permissions: jobs: linux: - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -70,7 +70,7 @@ jobs: dist/conformance-linux windows: - runs-on: windows-2025 + runs-on: windows-2022 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -118,7 +118,7 @@ jobs: publish: if: ${{ github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish) }} needs: [linux, windows] - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 @@ -156,5 +156,6 @@ jobs: dist/RELEASE_STATUS.json \ dist/checksums.json \ dist/release-manifest.json \ + --target "$GITHUB_SHA" \ --title 'PortaPy 3.14.0' \ --notes-file dist/RELEASE_NOTES.md From 71bfe895d2eacd836ae575700e792375b1da84eb Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:15:35 -0500 Subject: [PATCH 029/450] Surface full-core probe runtime errors --- tests/test_full_core_probe.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_full_core_probe.py b/tests/test_full_core_probe.py index f4565273..625b13ad 100644 --- a/tests/test_full_core_probe.py +++ b/tests/test_full_core_probe.py @@ -8,5 +8,12 @@ def test_full_core_probe_executes_reference_abi_path() -> None: materialize_reference_entry() normalize_reference_abi_helpers() from portapy.native_full_core_probe import portapy_full_core_probe + from portapy.native_full_reference_entry import _runtimes - assert portapy_full_core_probe() == 42 + result = portapy_full_core_probe() + errors = [ + runtime.last_error() + for runtime in _runtimes + if runtime is not None and runtime.last_error() is not None + ] + assert result == 42, errors From af5e7ce5b9855ab8772f9c7b0ef985bfa2e0da00 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:15:47 -0500 Subject: [PATCH 030/450] Expose full-core probe runtime error --- src/portapy/native_full_core_probe.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/portapy/native_full_core_probe.py b/src/portapy/native_full_core_probe.py index 174d850e..f6628a3b 100644 --- a/src/portapy/native_full_core_probe.py +++ b/src/portapy/native_full_core_probe.py @@ -89,6 +89,7 @@ def fail(): """ status = _portapy_exec_span_impl(runtime, source, len(source)) if status != PORTAPY_OK: + print("FULL CORE PROBE ERROR", instance.last_error()) return -1 handle = _portapy_get_global_span_impl(runtime, "answer", 6) if _portapy_value_get_kind_impl(runtime, handle) != PORTAPY_VALUE_INT: From 01e7074ae123fcd89b218dd59a0ae00f6ed03ecb Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:18:41 -0500 Subject: [PATCH 031/450] Seed every runtime with the standalone builtin table --- src/portapy/reference_api.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/portapy/reference_api.py b/src/portapy/reference_api.py index c9a01120..ef481430 100644 --- a/src/portapy/reference_api.py +++ b/src/portapy/reference_api.py @@ -11,6 +11,7 @@ import traceback from .core.frontend import compile_source +from .core.loader import default_builtins from .core.vm import VirtualMachine @@ -56,8 +57,10 @@ class _Slot: class Runtime: def __init__(self) -> None: self._vm = VirtualMachine() - self._globals: dict[str, object] = {} - self._globals.update({"__name__": "__main__", "__package__": "", "__doc__": None}) + self._globals: dict[str, object] = default_builtins() + self._globals.update( + {"__name__": "__main__", "__package__": "", "__doc__": None} + ) self._values: dict[int, _Slot] = {} self._next = 1 self._eval_counter = 0 From d0b1316f3a679af09482ec2df36991c17d89625c Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:18:45 -0500 Subject: [PATCH 032/450] Seed builtin names in every full Runtime --- tools/normalize_full_reference_abi_helpers.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/normalize_full_reference_abi_helpers.py b/tools/normalize_full_reference_abi_helpers.py index 1dfd97fd..2b2ed84a 100644 --- a/tools/normalize_full_reference_abi_helpers.py +++ b/tools/normalize_full_reference_abi_helpers.py @@ -3,9 +3,9 @@ The verified source payload originally named several Python-authored functions with the ``_portapy_cabi_`` prefix. PortaPy's linker layer reserves those names for register-preserving assembly adapters. This pass renames the underlying -implementations, removes redundant forwarding wrappers, installs host-owned -module resolution for every runtime, and preserves UTF-8 C span semantics at -the Python-authored ABI boundary. +implementations, removes redundant forwarding wrappers, installs explicit +builtins and host-owned module resolution for every runtime, and preserves +UTF-8 C span semantics at the Python-authored ABI boundary. """ from __future__ import annotations @@ -74,6 +74,7 @@ def __call__(self, name: str) -> object: _RUNTIME_CREATE_SOURCE = ''' instance = Runtime() +instance._vm._seed_builtins(instance._globals) instance.set_global("__pyinbin_import__", _PortaPyImportLoader(instance)) _runtimes.append(instance) _set_status(PORTAPY_OK) @@ -188,16 +189,19 @@ def main() -> int: "_PortaPyImportLoader" in classes and "__pyinbin_import__" in runtime_create_text ) - if missing or stale or unsafe_spans or not loader_ready: + builtins_ready = "_seed_builtins" in runtime_create_text + if missing or stale or unsafe_spans or not loader_ready or not builtins_ready: raise RuntimeError( "full Runtime ABI helper normalization failed; " f"missing={missing}, stale={stale}, " - f"unsafe_utf8_spans={len(unsafe_spans)}, loader_ready={loader_ready}" + f"unsafe_utf8_spans={len(unsafe_spans)}, " + f"loader_ready={loader_ready}, builtins_ready={builtins_ready}" ) print( "NORMALIZED FULL RUNTIME ABI HELPERS", len(_RENAME), len(_DROP), + "BUILTINS", "IMPORT_LOADER", ) return 0 From 1bd1552c92ec0eb61a73a35adababca41e90f9ed Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:19:09 -0500 Subject: [PATCH 033/450] Test builtin seeding in full Runtime normalization --- tests/test_full_reference_abi_normalization.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_full_reference_abi_normalization.py b/tests/test_full_reference_abi_normalization.py index 07c1e2e3..6c65a9b6 100644 --- a/tests/test_full_reference_abi_normalization.py +++ b/tests/test_full_reference_abi_normalization.py @@ -7,7 +7,7 @@ from tools import normalize_full_reference_abi_helpers as normalizer -def test_full_reference_normalization_installs_import_loader_and_utf8_spans( +def test_full_reference_normalization_installs_runtime_support( tmp_path: Path, monkeypatch, ) -> None: @@ -35,6 +35,7 @@ def test_full_reference_normalization_installs_import_loader_and_utf8_spans( and node.name == "_portapy_runtime_create_impl" ) runtime_source = ast.unparse(runtime_create) + assert "instance._vm._seed_builtins(instance._globals)" in runtime_source assert "_PortaPyImportLoader(instance)" in runtime_source assert "__pyinbin_import__" in runtime_source From c1a86689c3946c65c6ab9d018b5cdd37439da0f5 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:21:16 -0500 Subject: [PATCH 034/450] Upload complete native build logs --- .github/workflows/release-3.14.0.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-3.14.0.yml b/.github/workflows/release-3.14.0.yml index a4b7ebab..1daf5d5c 100644 --- a/.github/workflows/release-3.14.0.yml +++ b/.github/workflows/release-3.14.0.yml @@ -38,10 +38,13 @@ jobs: 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - name: Build full libportapy.so run: | + mkdir -p dist + set -o pipefail python tools/build_native_typed.py \ --target linux \ --output dist/libportapy.so \ - --work-dir dist/build-linux + --work-dir dist/build-linux \ + 2>&1 | tee dist/build-linux.log - name: Validate Linux stable artifact run: | readelf -dW dist/libportapy.so | tee dist/linux-dynamic-section.txt @@ -63,6 +66,7 @@ jobs: with: name: portapy-linux-3.14.0 path: | + dist/build-linux.log dist/libportapy.so dist/libportapy.so.json dist/linux-dynamic-section.txt @@ -86,11 +90,14 @@ jobs: - name: Build full portapy.dll shell: powershell run: | + New-Item -ItemType Directory -Force dist | Out-Null python tools/build_native_typed.py ` --target windows ` --output dist/portapy.dll ` - --work-dir dist/build-windows - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + --work-dir dist/build-windows ` + 2>&1 | Tee-Object -FilePath dist/build-windows.log + $buildStatus = $LASTEXITCODE + if ($buildStatus -ne 0) { exit $buildStatus } - name: Validate Windows stable artifact shell: powershell run: | @@ -110,6 +117,7 @@ jobs: with: name: portapy-windows-3.14.0 path: | + dist/build-windows.log dist/portapy.dll dist/portapy.dll.json dist/windows-imports.txt From fdcf847c6cbdc72d36178d2f1e81db45587509ee Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:24:27 -0500 Subject: [PATCH 035/450] Make native AST arg normalization idempotent --- tools/combine_full_core_native_parser.py | 49 ++++++++++++++++-------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/tools/combine_full_core_native_parser.py b/tools/combine_full_core_native_parser.py index bed39949..487a050e 100644 --- a/tools/combine_full_core_native_parser.py +++ b/tools/combine_full_core_native_parser.py @@ -86,6 +86,8 @@ def _prepare_runtime(module: ast.Module) -> list[ast.stmt]: def _rename_ast_arg_class(module: ast.Module) -> None: class_count = 0 call_count = 0 + existing_class_count = 0 + existing_call_count = 0 class _AnnotationRenamer(ast.NodeTransformer): def visit_Name(self, node: ast.Name) -> ast.AST: @@ -95,16 +97,18 @@ def visit_Name(self, node: ast.Name) -> ast.AST: renamer = _AnnotationRenamer() for node in ast.walk(module): - if isinstance(node, ast.ClassDef) and node.name == "arg": - node.name = "AstArg" - class_count += 1 - elif ( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id == "arg" - ): - node.func.id = "AstArg" - call_count += 1 + if isinstance(node, ast.ClassDef): + if node.name == "arg": + node.name = "AstArg" + class_count += 1 + elif node.name == "AstArg": + existing_class_count += 1 + elif isinstance(node, ast.Call) and isinstance(node.func, ast.Name): + if node.func.id == "arg": + node.func.id = "AstArg" + call_count += 1 + elif node.func.id == "AstArg": + existing_call_count += 1 if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): arguments = [ @@ -124,12 +128,27 @@ def visit_Name(self, node: ast.Name) -> ast.AST: elif isinstance(node, ast.AnnAssign): node.annotation = renamer.visit(node.annotation) - if class_count != 1 or call_count < 1: - raise RuntimeError( - "native AST arg rename failed: " - f"classes={class_count}, calls={call_count}" + if class_count == 1 and call_count >= 1: + print("RENAMED NATIVE AST ARG CLASS", class_count, call_count) + return + if ( + class_count == 0 + and call_count == 0 + and existing_class_count == 1 + and existing_call_count >= 1 + ): + print( + "PRESERVED RENAMED NATIVE AST ARG CLASS", + existing_class_count, + existing_call_count, ) - print("RENAMED NATIVE AST ARG CLASS", class_count, call_count) + return + raise RuntimeError( + "native AST arg rename failed: " + f"classes={class_count}, calls={call_count}, " + f"existing_classes={existing_class_count}, " + f"existing_calls={existing_call_count}" + ) def _prepare_bridge(module: ast.Module) -> list[ast.stmt]: From 7893b9fbf939f010b814a8dd80e3336c76400be2 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:24:37 -0500 Subject: [PATCH 036/450] Add concise native compiler diagnostic --- .github/workflows/native-build-diagnostic.yml | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/native-build-diagnostic.yml diff --git a/.github/workflows/native-build-diagnostic.yml b/.github/workflows/native-build-diagnostic.yml new file mode 100644 index 00000000..50c332b4 --- /dev/null +++ b/.github/workflows/native-build-diagnostic.yml @@ -0,0 +1,63 @@ +name: Native build diagnostic + +on: + pull_request: + branches: [main] + workflow_dispatch: + +concurrency: + group: native-build-diagnostic-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build-linux: + runs-on: ubuntu-22.04 + outputs: + status: ${{ steps.build.outputs.status }} + tail: ${{ steps.build.outputs.tail }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install quietly + shell: bash + run: | + sudo apt-get update > /tmp/install.log 2>&1 + sudo apt-get install --yes nasm gcc binutils >> /tmp/install.log 2>&1 + python -m pip install -e '.[test]' >> /tmp/install.log 2>&1 + python -m pip install --no-cache-dir --force-reinstall \ + 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' \ + >> /tmp/install.log 2>&1 + - name: Capture compiler tail + id: build + shell: bash + run: | + set +e + python tools/build_native_typed.py \ + --target linux \ + --output /tmp/libportapy.so \ + --work-dir /tmp/build-linux \ + > /tmp/build.log 2>&1 + status=$? + echo "status=$status" >> "$GITHUB_OUTPUT" + echo 'tail<> "$GITHUB_OUTPUT" + tail -n 90 /tmp/build.log >> "$GITHUB_OUTPUT" + echo 'PORTAPY_BUILD_TAIL' >> "$GITHUB_OUTPUT" + exit 0 + + report-linux: + needs: build-linux + runs-on: ubuntu-22.04 + steps: + - name: Report compiler tail + env: + BUILD_TAIL: ${{ needs.build-linux.outputs.tail }} + BUILD_STATUS: ${{ needs.build-linux.outputs.status }} + shell: bash + run: | + printf '%s\n' "$BUILD_TAIL" + exit "$BUILD_STATUS" From 1195b3afeee793990ef8a7c635bd38b477c65fb6 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:27:54 -0500 Subject: [PATCH 037/450] Bridge extended semantics across keyword parser ordering --- ...ize_full_core_extended_semantics_compat.py | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 tools/normalize_full_core_extended_semantics_compat.py diff --git a/tools/normalize_full_core_extended_semantics_compat.py b/tools/normalize_full_core_extended_semantics_compat.py new file mode 100644 index 00000000..13ed43f7 --- /dev/null +++ b/tools/normalize_full_core_extended_semantics_compat.py @@ -0,0 +1,73 @@ +"""Run extended-semantics normalization across current parser pass ordering. + +``normalize_full_core_keyword_calls`` replaces ``_parse_call_args`` before the +extended-semantics pass runs. The latter historically tried to rewrite the +old method's explicit ``*`` and ``**`` branches even though it subsequently +replaces the whole method with the native bootstrap implementation. This +wrapper permits those two stale textual rewrites to be absent only when the +current method contains no starred AST construction at all. +""" +from __future__ import annotations + +from tools import normalize_full_core_extended_semantics as _base + + +_OPTIONAL_LABELS = { + "keyword unpack parser": "_npr_ast_nodes_DoubleStarred", + "positional unpack parser": "_npr_ast_nodes_Starred", +} + + +def _call_argument_method(source: str) -> str: + signature = " def _parse_call_args(self):" + next_signature = "\n def _parse_tuple_rhs(self):" + start = source.find(signature) + if start < 0: + raise RuntimeError("call argument compatibility: method start not found") + end = source.find(next_signature, start + len(signature)) + if end < 0: + raise RuntimeError("call argument compatibility: next method not found") + return source[start:end] + + +def _compatible_replace( + source: str, + old: str, + new: str, + *, + label: str, + expected: int = 1, +) -> str: + count = source.count(old) + marker = _OPTIONAL_LABELS.get(label) + if marker is not None and count == 0: + method = _call_argument_method(source) + if marker in method: + raise RuntimeError( + f"{label}: stale rewrite missing while {marker} remains" + ) + print("SKIPPED", label, "call parser already normalized") + return source + return _original_replace( + source, + old, + new, + label=label, + expected=expected, + ) + + +_original_replace = _base._replace + + +def main() -> int: + previous = _base._replace + _base._replace = _compatible_replace + try: + return _base.main() + finally: + _base._replace = previous + + +if __name__ == "__main__": + raise SystemExit(main()) From 2c29a3c26f11cc790db3446b513c8686f75e3208 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:28:09 -0500 Subject: [PATCH 038/450] Preserve call parser shape between native normalization passes --- tools/normalize_full_core_calls_closures.py | 30 +++++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/tools/normalize_full_core_calls_closures.py b/tools/normalize_full_core_calls_closures.py index 8e4a42c2..28f72e12 100644 --- a/tools/normalize_full_core_calls_closures.py +++ b/tools/normalize_full_core_calls_closures.py @@ -108,14 +108,34 @@ def _full_core_parser_collect_expr_names(node: object, result: list[str]) -> Non _CALL_ARGS = ''' def _parse_call_args(self): - args: list[object] = [] - kwargs: list[object] = [] - while not self._check('OP', ')'): - value = self._parse_expr() - args.append(value) + args: list = [] + kwargs: list = [] + if self._check('OP', ')'): + return (args, kwargs) + while True: + if self._check('NAME') and self._peek(1).kind == 'OP' and (self._peek(1).value == '='): + name = self._eat().value + self._eat() + kwargs.append((name, self._parse_expr())) + elif self._check('OP', '**'): + star_pos = self._eat().pos + args.append(_npr_ast_nodes_DoubleStarred(value=self._parse_expr(), pos=star_pos)) + else: + if kwargs: + raise _npr_errors_ParseError('positional argument follows keyword argument', self._peek().pos) + if self._check('OP', '*'): + star_pos = self._eat().pos + args.append(_npr_ast_nodes_Starred(value=self._parse_expr(), pos=star_pos)) + else: + arg = self._parse_expr() + if self._check('KEYWORD', 'for'): + arg = self._parse_comprehension_tail(arg, arg.pos) + args.append(arg) if not self._check('OP', ','): break self._eat() + if self._check('OP', ')'): + break return (args, kwargs) ''' From 50101614abc30dabb8a1f2bcd69ef9026266bb3f Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:29:05 -0500 Subject: [PATCH 039/450] Use parser-order compatible extended semantics pass --- tools/normalize_full_core_validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/normalize_full_core_validation.py b/tools/normalize_full_core_validation.py index a220169f..e21de71b 100644 --- a/tools/normalize_full_core_validation.py +++ b/tools/normalize_full_core_validation.py @@ -11,7 +11,7 @@ ) from tools.normalize_full_core_closures import main as normalize_closures from tools.normalize_full_core_collections import main as normalize_collections -from tools.normalize_full_core_extended_semantics import ( +from tools.normalize_full_core_extended_semantics_compat import ( main as normalize_extended_semantics, ) from tools.normalize_full_core_keyword_calls import ( From e2e6cebf21adc6ca49aca1c5fcb977640025c9f1 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:32:15 -0500 Subject: [PATCH 040/450] Recognize already-normalized exception VM forms --- ...ize_full_core_extended_semantics_compat.py | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/tools/normalize_full_core_extended_semantics_compat.py b/tools/normalize_full_core_extended_semantics_compat.py index 13ed43f7..d2bb57f1 100644 --- a/tools/normalize_full_core_extended_semantics_compat.py +++ b/tools/normalize_full_core_extended_semantics_compat.py @@ -1,22 +1,26 @@ -"""Run extended-semantics normalization across current parser pass ordering. - -``normalize_full_core_keyword_calls`` replaces ``_parse_call_args`` before the -extended-semantics pass runs. The latter historically tried to rewrite the -old method's explicit ``*`` and ``**`` branches even though it subsequently -replaces the whole method with the native bootstrap implementation. This -wrapper permits those two stale textual rewrites to be absent only when the -current method contains no starred AST construction at all. +"""Run extended-semantics normalization across current pass ordering. + +Several earlier native-bootstrap passes now produce the exact target form that +``normalize_full_core_extended_semantics`` historically created itself. This +wrapper skips only those stale textual rewrites whose replacement semantics are +already present, while retaining the original fail-closed behavior everywhere +else. """ from __future__ import annotations from tools import normalize_full_core_extended_semantics as _base -_OPTIONAL_LABELS = { +_CALL_PARSER_MARKERS = { "keyword unpack parser": "_npr_ast_nodes_DoubleStarred", "positional unpack parser": "_npr_ast_nodes_Starred", } +_ALREADY_NORMALIZED_MARKERS = { + "dynamic exception reraising": "RuntimeError: exception did not match handler", + "exception stack extension": "frame.stack.append(matched)", +} + def _call_argument_method(source: str) -> str: signature = " def _parse_call_args(self):" @@ -39,15 +43,25 @@ def _compatible_replace( expected: int = 1, ) -> str: count = source.count(old) - marker = _OPTIONAL_LABELS.get(label) - if marker is not None and count == 0: + call_marker = _CALL_PARSER_MARKERS.get(label) + if call_marker is not None and count == 0: method = _call_argument_method(source) - if marker in method: + if call_marker in method: raise RuntimeError( - f"{label}: stale rewrite missing while {marker} remains" + f"{label}: stale rewrite missing while {call_marker} remains" ) print("SKIPPED", label, "call parser already normalized") return source + + target_marker = _ALREADY_NORMALIZED_MARKERS.get(label) + if target_marker is not None and count == 0: + if target_marker not in source: + raise RuntimeError( + f"{label}: neither source nor normalized target form is present" + ) + print("SKIPPED", label, "target form already present") + return source + return _original_replace( source, old, From e5dadef4f862832de425b0cb621b04ffb3011a60 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:32:57 -0500 Subject: [PATCH 041/450] Recognize normalized VM context-manager paths --- tools/normalize_full_core_extended_semantics_compat.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/normalize_full_core_extended_semantics_compat.py b/tools/normalize_full_core_extended_semantics_compat.py index d2bb57f1..84c7f2ea 100644 --- a/tools/normalize_full_core_extended_semantics_compat.py +++ b/tools/normalize_full_core_extended_semantics_compat.py @@ -19,6 +19,7 @@ _ALREADY_NORMALIZED_MARKERS = { "dynamic exception reraising": "RuntimeError: exception did not match handler", "exception stack extension": "frame.stack.append(matched)", + "context-manager exception forwarding": "exit_args.append(exc_type)", } @@ -55,11 +56,13 @@ def _compatible_replace( target_marker = _ALREADY_NORMALIZED_MARKERS.get(label) if target_marker is not None and count == 0: - if target_marker not in source: + marker_count = source.count(target_marker) + if marker_count != 1: raise RuntimeError( - f"{label}: neither source nor normalized target form is present" + f"{label}: neither unique source nor normalized target form is present; " + f"target matches={marker_count}" ) - print("SKIPPED", label, "target form already present") + print("PRESERVED", label, marker_count) return source return _original_replace( From c321ff97d5ecc1b624895b5bc8ddbea10dcd06a4 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:35:15 -0500 Subject: [PATCH 042/450] Rewrite compact native exception dispatch safely --- ...ize_full_core_extended_semantics_compat.py | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/tools/normalize_full_core_extended_semantics_compat.py b/tools/normalize_full_core_extended_semantics_compat.py index 84c7f2ea..aa0be799 100644 --- a/tools/normalize_full_core_extended_semantics_compat.py +++ b/tools/normalize_full_core_extended_semantics_compat.py @@ -1,10 +1,9 @@ """Run extended-semantics normalization across current pass ordering. -Several earlier native-bootstrap passes now produce the exact target form that -``normalize_full_core_extended_semantics`` historically created itself. This -wrapper skips only those stale textual rewrites whose replacement semantics are -already present, while retaining the original fail-closed behavior everywhere -else. +Several earlier native-bootstrap passes now produce either the exact target form +or a compact equivalent of source that ``normalize_full_core_extended_semantics`` +historically rewrote. This wrapper recognizes only those verified shapes while +retaining the original fail-closed behavior everywhere else. """ from __future__ import annotations @@ -17,11 +16,19 @@ } _ALREADY_NORMALIZED_MARKERS = { - "dynamic exception reraising": "RuntimeError: exception did not match handler", "exception stack extension": "frame.stack.append(matched)", "context-manager exception forwarding": "exit_args.append(exc_type)", } +_DYNAMIC_EXCEPTION_COMPACT = ''' if not self._exception_matches(value, expected): + if isinstance(value, (BaseException, PyException)): raise value + _raise_typed("RuntimeError: invalid exception value")''' + +_DYNAMIC_EXCEPTION_TARGET = ''' if not self._exception_matches(value, expected): + if isinstance(value, (BaseException, PyException)): + _raise_typed("RuntimeError: exception did not match handler") + _raise_typed("RuntimeError: invalid exception value")''' + def _call_argument_method(source: str) -> str: signature = " def _parse_call_args(self):" @@ -54,6 +61,20 @@ def _compatible_replace( print("SKIPPED", label, "call parser already normalized") return source + if label == "dynamic exception reraising" and count == 0: + compact_count = source.count(_DYNAMIC_EXCEPTION_COMPACT) + if compact_count != 1: + raise RuntimeError( + "dynamic exception reraising: neither canonical nor compact " + f"source form is unique; compact matches={compact_count}" + ) + print("REPLACED", label, compact_count, "compact form") + return source.replace( + _DYNAMIC_EXCEPTION_COMPACT, + _DYNAMIC_EXCEPTION_TARGET, + 1, + ) + target_marker = _ALREADY_NORMALIZED_MARKERS.get(label) if target_marker is not None and count == 0: marker_count = source.count(target_marker) From 1e6e1ee25d2926ec5e5d7cac0fa215af60e211aa Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:37:09 -0500 Subject: [PATCH 043/450] Accept compact native collection bootstrap --- tools/normalize_full_core_collections.py | 38 +++++++++++++++++------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/tools/normalize_full_core_collections.py b/tools/normalize_full_core_collections.py index 4ad82f0c..76de8730 100644 --- a/tools/normalize_full_core_collections.py +++ b/tools/normalize_full_core_collections.py @@ -6,16 +6,20 @@ PATH = Path("src/portapy/core/vm.py") - -def main() -> int: - source = PATH.read_text(encoding="utf-8") - old = ''' elif op in (Op.BUILD_TUPLE, Op.BUILD_SET): +_CANONICAL_BOOTSTRAP = ''' elif op in (Op.BUILD_TUPLE, Op.BUILD_SET): if len(frame.stack) < instr.arg: _raise_typed("RuntimeError: collection stack underflow") values = _full_core_probe_pop_tail(frame.stack, instr.arg) frame.stack.append(None) ''' - new = ''' elif op in (Op.BUILD_TUPLE, Op.BUILD_SET): + +_COMPACT_BOOTSTRAP = ''' elif op in (Op.BUILD_TUPLE, Op.BUILD_SET): + if len(frame.stack) < instr.arg: _raise_typed("RuntimeError: collection stack underflow") + values = _full_core_probe_pop_tail(frame.stack, instr.arg) + frame.stack.append(None) +''' + +_RESTORED = ''' elif op in (Op.BUILD_TUPLE, Op.BUILD_SET): if len(frame.stack) < instr.arg: _raise_typed("RuntimeError: collection stack underflow") values = _full_core_probe_pop_tail(frame.stack, instr.arg) @@ -24,13 +28,27 @@ def main() -> int: else: frame.stack.append(set(values)) ''' - count = source.count(old) - if count != 1: + + +def main() -> int: + source = PATH.read_text(encoding="utf-8") + matches = [ + candidate + for candidate in (_CANONICAL_BOOTSTRAP, _COMPACT_BOOTSTRAP) + if source.count(candidate) == 1 + ] + if len(matches) != 1: + counts = { + "canonical": source.count(_CANONICAL_BOOTSTRAP), + "compact": source.count(_COMPACT_BOOTSTRAP), + } raise RuntimeError( - f"native tuple/set bootstrap: expected 1 match, found {count}" + "native tuple/set bootstrap: expected one verified source form, " + f"found {counts}" ) - PATH.write_text(source.replace(old, new, 1), encoding="utf-8") - print("RESTORED NATIVE TUPLE AND SET CONSTRUCTION", count) + source = source.replace(matches[0], _RESTORED, 1) + PATH.write_text(source, encoding="utf-8") + print("RESTORED NATIVE TUPLE AND SET CONSTRUCTION", 1) return 0 From fad124f81fef94c6298d5dca698ef1dff09d6a3b Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:37:21 -0500 Subject: [PATCH 044/450] Test native collection normalization variants --- tests/test_normalize_full_core_collections.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/test_normalize_full_core_collections.py diff --git a/tests/test_normalize_full_core_collections.py b/tests/test_normalize_full_core_collections.py new file mode 100644 index 00000000..7af40704 --- /dev/null +++ b/tests/test_normalize_full_core_collections.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tools import normalize_full_core_collections as collections + + +def _run(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, source: str) -> str: + path = tmp_path / "vm.py" + path.write_text(source, encoding="utf-8") + monkeypatch.setattr(collections, "PATH", path) + assert collections.main() == 0 + return path.read_text(encoding="utf-8") + + +def test_restores_canonical_collection_bootstrap( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + result = _run(tmp_path, monkeypatch, collections._CANONICAL_BOOTSTRAP) + assert result == collections._RESTORED + + +def test_restores_compact_collection_bootstrap( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + result = _run(tmp_path, monkeypatch, collections._COMPACT_BOOTSTRAP) + assert result == collections._RESTORED + + +def test_rejects_missing_or_ambiguous_collection_bootstrap( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "vm.py" + path.write_text( + collections._CANONICAL_BOOTSTRAP + collections._COMPACT_BOOTSTRAP, + encoding="utf-8", + ) + monkeypatch.setattr(collections, "PATH", path) + with pytest.raises(RuntimeError, match="expected one verified source form"): + collections.main() From 13500a5e786f8642a982e71e7dd833ebd287d6fe Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:39:17 -0500 Subject: [PATCH 045/450] Run complete native normalization sequence in canonical builder --- tools/build_native_full_runtime.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tools/build_native_full_runtime.py b/tools/build_native_full_runtime.py index 193e0013..5cc84040 100644 --- a/tools/build_native_full_runtime.py +++ b/tools/build_native_full_runtime.py @@ -17,6 +17,10 @@ from tools.elf_runtime_abi import fix_linux_runtime_abi from tools.nasm_direct_float_abi import append_direct_float_abi from tools.native_surface import public_exports +from tools.normalize_full_core_lambdas import main as normalize_lambdas +from tools.normalize_full_core_native_semantics import main as normalize_native_semantics +from tools.normalize_full_core_opcode_maps import main as normalize_opcode_maps +from tools.normalize_full_core_probe import main as normalize_probe from tools.normalize_full_core_validation import main as normalize_full_runtime from tools.python_surface import PYTHON_MODULE_EXPORTS @@ -24,6 +28,21 @@ SOURCE = REPOSITORY_ROOT / "src" / "portapy" / "native_full_reference_entry.py" +def _prepare_full_runtime_sources() -> None: + """Apply the verified full-core native normalization sequence once. + + This is the same ordering used by the original Linux/Windows full-core + probe workflow. Keeping it in the canonical builder makes a direct + ``build_native_typed.py`` invocation self-contained instead of depending + on CI having pre-mutated the checkout. + """ + normalize_probe() + normalize_lambdas() + normalize_native_semantics() + normalize_opcode_maps() + normalize_full_runtime() + + def _install_full_runtime_transforms() -> None: original_transform = base_build._transform_assembly @@ -73,7 +92,7 @@ def build_full_runtime( normalize: bool = True, ) -> dict[str, object]: if normalize: - normalize_full_runtime() + _prepare_full_runtime_sources() _install_full_runtime_transforms() metadata = base_build.build_native( From 2e0957e9deeb23eeab7973033e821453c527b52e Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:39:23 -0500 Subject: [PATCH 046/450] Make full Runtime normalization self-contained --- tools/normalize_full_core_validation.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tools/normalize_full_core_validation.py b/tools/normalize_full_core_validation.py index e21de71b..6160880a 100644 --- a/tools/normalize_full_core_validation.py +++ b/tools/normalize_full_core_validation.py @@ -1,4 +1,4 @@ -"""Apply final validation-oriented rewrites to the native full-core probe.""" +"""Apply the complete native full-core normalization pipeline.""" from __future__ import annotations from pathlib import Path @@ -17,8 +17,12 @@ from tools.normalize_full_core_keyword_calls import ( main as normalize_keyword_calls, ) +from tools.normalize_full_core_lambdas import main as normalize_lambdas from tools.normalize_full_core_native_parser import main as normalize_native_parser +from tools.normalize_full_core_native_semantics import main as normalize_native_semantics +from tools.normalize_full_core_opcode_maps import main as normalize_opcode_maps from tools.normalize_full_core_pattern_slices import main as normalize_pattern_slices +from tools.normalize_full_core_probe import main as normalize_probe from tools.normalize_full_reference_abi_helpers import ( main as normalize_reference_abi_helpers, ) @@ -60,6 +64,14 @@ def _normalize_opcode_validation() -> None: def main() -> int: + # These four passes were historically run only by the probe workflow. + # Keeping them here makes clean production builds use the same proven + # compiler-safe source preparation as the full-core transition probe. + normalize_probe() + normalize_lambdas() + normalize_native_semantics() + normalize_opcode_maps() + materialize_reference_entry() normalize_reference_abi_helpers() normalize_native_parser() From 56d23c921f1175e6b0c7ec38fadce2e400d54870 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:39:36 -0500 Subject: [PATCH 047/450] Normalize collection bootstrap structurally --- tools/normalize_full_core_collections.py | 61 ++++++++++++++---------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/tools/normalize_full_core_collections.py b/tools/normalize_full_core_collections.py index 76de8730..50693f0f 100644 --- a/tools/normalize_full_core_collections.py +++ b/tools/normalize_full_core_collections.py @@ -6,48 +6,57 @@ PATH = Path("src/portapy/core/vm.py") +_BLOCK_START = " elif op in (Op.BUILD_TUPLE, Op.BUILD_SET):\n" +_BLOCK_END = " elif op is Op.GET_ITEM:\n" +_BOOTSTRAP_VALUE = " frame.stack.append(None)" +_RESTORED_VALUE = ''' if op is Op.BUILD_TUPLE: + frame.stack.append(tuple(values)) + else: + frame.stack.append(set(values))''' + _CANONICAL_BOOTSTRAP = ''' elif op in (Op.BUILD_TUPLE, Op.BUILD_SET): if len(frame.stack) < instr.arg: _raise_typed("RuntimeError: collection stack underflow") values = _full_core_probe_pop_tail(frame.stack, instr.arg) frame.stack.append(None) + elif op is Op.GET_ITEM: ''' _COMPACT_BOOTSTRAP = ''' elif op in (Op.BUILD_TUPLE, Op.BUILD_SET): if len(frame.stack) < instr.arg: _raise_typed("RuntimeError: collection stack underflow") - values = _full_core_probe_pop_tail(frame.stack, instr.arg) + values = frame.stack[-instr.arg:] if instr.arg else [] + if instr.arg: del frame.stack[-instr.arg:] frame.stack.append(None) + elif op is Op.GET_ITEM: ''' -_RESTORED = ''' elif op in (Op.BUILD_TUPLE, Op.BUILD_SET): - if len(frame.stack) < instr.arg: - _raise_typed("RuntimeError: collection stack underflow") - values = _full_core_probe_pop_tail(frame.stack, instr.arg) - if op is Op.BUILD_TUPLE: - frame.stack.append(tuple(values)) - else: - frame.stack.append(set(values)) -''' + +def _collection_block(source: str) -> tuple[int, int, str]: + if source.count(_BLOCK_START) != 1: + raise RuntimeError( + "native tuple/set bootstrap: expected one BUILD_TUPLE/BUILD_SET block" + ) + start = source.find(_BLOCK_START) + end = source.find(_BLOCK_END, start + len(_BLOCK_START)) + if end < 0: + raise RuntimeError("native tuple/set bootstrap: GET_ITEM boundary not found") + block = source[start:end] + if "collection stack underflow" not in block or "values =" not in block: + raise RuntimeError("native tuple/set bootstrap: required safety checks are missing") + if block.count(_BOOTSTRAP_VALUE) != 1: + raise RuntimeError( + "native tuple/set bootstrap: expected one placeholder collection result" + ) + if "tuple(values)" in block or "set(values)" in block: + raise RuntimeError("native tuple/set bootstrap: block is already partially restored") + return start, end, block def main() -> int: source = PATH.read_text(encoding="utf-8") - matches = [ - candidate - for candidate in (_CANONICAL_BOOTSTRAP, _COMPACT_BOOTSTRAP) - if source.count(candidate) == 1 - ] - if len(matches) != 1: - counts = { - "canonical": source.count(_CANONICAL_BOOTSTRAP), - "compact": source.count(_COMPACT_BOOTSTRAP), - } - raise RuntimeError( - "native tuple/set bootstrap: expected one verified source form, " - f"found {counts}" - ) - source = source.replace(matches[0], _RESTORED, 1) - PATH.write_text(source, encoding="utf-8") + start, end, block = _collection_block(source) + restored = block.replace(_BOOTSTRAP_VALUE, _RESTORED_VALUE, 1) + PATH.write_text(source[:start] + restored + source[end:], encoding="utf-8") print("RESTORED NATIVE TUPLE AND SET CONSTRUCTION", 1) return 0 From 2609b4d18bc1548ad5cfd5f8f10a2f5a4d82ba0d Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:39:50 -0500 Subject: [PATCH 048/450] Test structural collection normalization --- tests/test_normalize_full_core_collections.py | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/tests/test_normalize_full_core_collections.py b/tests/test_normalize_full_core_collections.py index 7af40704..b449a25c 100644 --- a/tests/test_normalize_full_core_collections.py +++ b/tests/test_normalize_full_core_collections.py @@ -15,21 +15,29 @@ def _run(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, source: str) -> str: return path.read_text(encoding="utf-8") +def _assert_restored(result: str) -> None: + assert "frame.stack.append(None)" not in result + assert "frame.stack.append(tuple(values))" in result + assert "frame.stack.append(set(values))" in result + + def test_restores_canonical_collection_bootstrap( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - result = _run(tmp_path, monkeypatch, collections._CANONICAL_BOOTSTRAP) - assert result == collections._RESTORED + _assert_restored( + _run(tmp_path, monkeypatch, collections._CANONICAL_BOOTSTRAP) + ) def test_restores_compact_collection_bootstrap( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: result = _run(tmp_path, monkeypatch, collections._COMPACT_BOOTSTRAP) - assert result == collections._RESTORED + _assert_restored(result) + assert "if instr.arg: del frame.stack[-instr.arg:]" in result -def test_rejects_missing_or_ambiguous_collection_bootstrap( +def test_rejects_ambiguous_collection_bootstrap( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: path = tmp_path / "vm.py" @@ -38,5 +46,18 @@ def test_rejects_missing_or_ambiguous_collection_bootstrap( encoding="utf-8", ) monkeypatch.setattr(collections, "PATH", path) - with pytest.raises(RuntimeError, match="expected one verified source form"): + with pytest.raises(RuntimeError, match="expected one BUILD_TUPLE/BUILD_SET block"): + collections.main() + + +def test_rejects_collection_block_without_placeholder( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = collections._CANONICAL_BOOTSTRAP.replace( + "frame.stack.append(None)", "frame.stack.append(values)" + ) + path = tmp_path / "vm.py" + path.write_text(source, encoding="utf-8") + monkeypatch.setattr(collections, "PATH", path) + with pytest.raises(RuntimeError, match="placeholder collection result"): collections.main() From e2b6e85e6cbf86b9a81861d6673b79e839cc8345 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:40:50 -0500 Subject: [PATCH 049/450] Run full native normalization pipeline exactly once --- tools/build_native_full_runtime.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/tools/build_native_full_runtime.py b/tools/build_native_full_runtime.py index 5cc84040..af666f31 100644 --- a/tools/build_native_full_runtime.py +++ b/tools/build_native_full_runtime.py @@ -17,10 +17,6 @@ from tools.elf_runtime_abi import fix_linux_runtime_abi from tools.nasm_direct_float_abi import append_direct_float_abi from tools.native_surface import public_exports -from tools.normalize_full_core_lambdas import main as normalize_lambdas -from tools.normalize_full_core_native_semantics import main as normalize_native_semantics -from tools.normalize_full_core_opcode_maps import main as normalize_opcode_maps -from tools.normalize_full_core_probe import main as normalize_probe from tools.normalize_full_core_validation import main as normalize_full_runtime from tools.python_surface import PYTHON_MODULE_EXPORTS @@ -29,17 +25,7 @@ def _prepare_full_runtime_sources() -> None: - """Apply the verified full-core native normalization sequence once. - - This is the same ordering used by the original Linux/Windows full-core - probe workflow. Keeping it in the canonical builder makes a direct - ``build_native_typed.py`` invocation self-contained instead of depending - on CI having pre-mutated the checkout. - """ - normalize_probe() - normalize_lambdas() - normalize_native_semantics() - normalize_opcode_maps() + """Apply the complete verified native normalization pipeline once.""" normalize_full_runtime() From 9dabb3a4641b9d1795e1dbe38ad72a8d319ca0f0 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:42:54 -0500 Subject: [PATCH 050/450] Expose full native compiler diagnostic --- .github/workflows/native-build-diagnostic.yml | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/native-build-diagnostic.yml b/.github/workflows/native-build-diagnostic.yml index 50c332b4..17c2cef3 100644 --- a/.github/workflows/native-build-diagnostic.yml +++ b/.github/workflows/native-build-diagnostic.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-22.04 outputs: status: ${{ steps.build.outputs.status }} - tail: ${{ steps.build.outputs.tail }} + report: ${{ steps.build.outputs.report }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -32,7 +32,7 @@ jobs: python -m pip install --no-cache-dir --force-reinstall \ 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' \ >> /tmp/install.log 2>&1 - - name: Capture compiler tail + - name: Capture compiler diagnostic id: build shell: bash run: | @@ -43,21 +43,32 @@ jobs: --work-dir /tmp/build-linux \ > /tmp/build.log 2>&1 status=$? + { + echo '=== FIRST 35 LINES ===' + head -n 35 /tmp/build.log + echo + echo '=== ERROR-LIKE LINES ===' + grep -Ein 'error|failed|failure|unsupported|traceback|exception|undefined|invalid' \ + /tmp/build.log | tail -n 120 || true + echo + echo '=== FINAL 120 LINES ===' + tail -n 120 /tmp/build.log + } > /tmp/report.log echo "status=$status" >> "$GITHUB_OUTPUT" - echo 'tail<> "$GITHUB_OUTPUT" - tail -n 90 /tmp/build.log >> "$GITHUB_OUTPUT" - echo 'PORTAPY_BUILD_TAIL' >> "$GITHUB_OUTPUT" + echo 'report<> "$GITHUB_OUTPUT" + cat /tmp/report.log >> "$GITHUB_OUTPUT" + echo 'PORTAPY_BUILD_REPORT' >> "$GITHUB_OUTPUT" exit 0 report-linux: needs: build-linux runs-on: ubuntu-22.04 steps: - - name: Report compiler tail + - name: Report compiler diagnostic env: - BUILD_TAIL: ${{ needs.build-linux.outputs.tail }} + BUILD_REPORT: ${{ needs.build-linux.outputs.report }} BUILD_STATUS: ${{ needs.build-linux.outputs.status }} shell: bash run: | - printf '%s\n' "$BUILD_TAIL" + printf '%s\n' "$BUILD_REPORT" exit "$BUILD_STATUS" From 27a7a29a29b045f34c1f5e671f97cd03e7173a44 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:44:29 -0500 Subject: [PATCH 051/450] Upload normalized native compiler input on failure --- .github/workflows/native-build-diagnostic.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/native-build-diagnostic.yml b/.github/workflows/native-build-diagnostic.yml index 17c2cef3..41cae7b1 100644 --- a/.github/workflows/native-build-diagnostic.yml +++ b/.github/workflows/native-build-diagnostic.yml @@ -59,6 +59,19 @@ jobs: cat /tmp/report.log >> "$GITHUB_OUTPUT" echo 'PORTAPY_BUILD_REPORT' >> "$GITHUB_OUTPUT" exit 0 + - name: Upload normalized compiler input + if: always() + uses: actions/upload-artifact@v4 + with: + name: native-build-diagnostic-source + path: | + /tmp/build.log + /tmp/report.log + src/portapy/native_full_reference_entry.py + src/portapy/core/frontend.py + src/portapy/core/vm.py + src/portapy/core/loader.py + src/portapy/core/bytecode.py report-linux: needs: build-linux From ae9ea98cfcdf18b97af8bb1b5c2c0c133eb45c24 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:47:38 -0500 Subject: [PATCH 052/450] Enable PortaPy builtin symbols in pinned compiler --- tools/normalize_full_core_lambdas.py | 160 +++++++++++++++++++++++++-- 1 file changed, 149 insertions(+), 11 deletions(-) diff --git a/tools/normalize_full_core_lambdas.py b/tools/normalize_full_core_lambdas.py index c13605ab..d862d10c 100644 --- a/tools/normalize_full_core_lambdas.py +++ b/tools/normalize_full_core_lambdas.py @@ -6,6 +6,7 @@ import re import asmpython +import asmpython._compiler.codegen as asmpython_codegen import asmpython._compiler.sema as asmpython_sema @@ -15,6 +16,7 @@ ASMPYTHON_ROOT = Path(asmpython.__file__).resolve().parent ASMPYTHON_STDLIB = ASMPYTHON_ROOT / "stdlib" ASMPYTHON_SEMA = Path(asmpython_sema.__file__).resolve() +ASMPYTHON_CODEGEN = Path(asmpython_codegen.__file__).resolve() _FSTRING_CONVERTER_ASSIGN = ( @@ -26,6 +28,30 @@ _FSTRING_CONVERTER_EMIT = "self.emit(Op.LOAD_CONST, self.constant(converter))" _FSTRING_NAME_EMIT = "self.emit(Op.LOAD_NAME, self.name_index(converter_name))" +_PORTAPY_EXCEPTION_PARENTS = { + "GeneratorExit": "BaseException", + "SyntaxError": "Exception", + "StopAsyncIteration": "Exception", + "ModuleNotFoundError": "ImportError", +} +_PORTAPY_EXCEPTION_IDS = { + "GeneratorExit": 22, + "SyntaxError": 23, + "StopAsyncIteration": 24, + "ModuleNotFoundError": 25, +} +_PORTAPY_TYPE_IDS = { + "bytes": -9, + "bytearray": -10, + "object": -11, + "type": -12, + "slice": -13, + "frozenset": -14, + "staticmethod": -15, + "classmethod": -16, + "property": -17, +} + def _normalize_ascii(source: str) -> tuple[str, int, int]: call_count = source.count("ascii(") @@ -43,20 +69,132 @@ def _normalize_ascii_file(path: Path, label: str) -> None: print(f"REPLACED {label} ASCII CONVERSIONS", conversion_count) -def _enable_compiler_ascii() -> None: - source = ASMPYTHON_SEMA.read_text(encoding="utf-8") - marker = ' "repr": (1, 1),\n' - if marker not in source: +def _insert_mapping_entries( + source: str, + *, + marker: str, + entries: dict[str, object], + label: str, +) -> str: + missing = [name for name in entries if f' "{name}":' not in source] + if not missing: + return source + if source.count(marker) != 1: + raise RuntimeError(f"{label}: expected one insertion marker") + lines = "".join(f' "{name}": {entries[name]!r},\n' for name in missing) + return source.replace(marker, lines + marker, 1) + + +def _insert_set_entries( + source: str, + *, + marker: str, + names: tuple[str, ...], + label: str, +) -> str: + missing = [name for name in names if f' "{name}",' not in source] + if not missing: + return source + if source.count(marker) != 1: + raise RuntimeError(f"{label}: expected one insertion marker") + lines = "".join(f' "{name}",\n' for name in missing) + return source.replace(marker, lines + marker, 1) + + +def _patch_compiler_runtime_symbols( + sema_source: str, + codegen_source: str, +) -> tuple[str, str]: + sema_source = _insert_set_entries( + sema_source, + marker=' "FileNotFoundError",\n})', + names=tuple(_PORTAPY_EXCEPTION_PARENTS), + label="asmpython builtin exceptions", + ) + sema_source = _insert_set_entries( + sema_source, + marker=' "int", "float", "str", "bool", "list", "dict", "tuple", "set",\n})', + names=tuple(_PORTAPY_TYPE_IDS), + label="asmpython builtin type names", + ) + codegen_source = _insert_mapping_entries( + codegen_source, + marker=' "StopIteration": "Exception",\n}', + entries=_PORTAPY_EXCEPTION_PARENTS, + label="asmpython exception parents", + ) + codegen_source = _insert_mapping_entries( + codegen_source, + marker=' "IOError": 19, # alias for OSError (same id)\n}', + entries=_PORTAPY_EXCEPTION_IDS, + label="asmpython exception ids", + ) + codegen_source = _insert_mapping_entries( + codegen_source, + marker=' "set": -8,\n}', + entries=_PORTAPY_TYPE_IDS, + label="asmpython builtin type ids", + ) + return sema_source, codegen_source + + +def _enable_compiler_builtins() -> None: + sema_source = ASMPYTHON_SEMA.read_text(encoding="utf-8") + ascii_marker = ' "repr": (1, 1),\n' + if ascii_marker not in sema_source: raise RuntimeError("asmpython semantic builtin table is missing repr") - if ' "ascii": (1, 1),\n' not in source: - source = source.replace(marker, marker + ' "ascii": (1, 1),\n', 1) - ASMPYTHON_SEMA.write_text(source, encoding="utf-8") + if ' "ascii": (1, 1),\n' not in sema_source: + sema_source = sema_source.replace( + ascii_marker, + ascii_marker + ' "ascii": (1, 1),\n', + 1, + ) + + codegen_source = ASMPYTHON_CODEGEN.read_text(encoding="utf-8") + sema_source, codegen_source = _patch_compiler_runtime_symbols( + sema_source, + codegen_source, + ) + ASMPYTHON_SEMA.write_text(sema_source, encoding="utf-8") + ASMPYTHON_CODEGEN.write_text(codegen_source, encoding="utf-8") + importlib.invalidate_caches() - reloaded = importlib.reload(asmpython_sema) - if "ascii" not in reloaded.BUILTINS: + reloaded_sema = importlib.reload(asmpython_sema) + reloaded_codegen = importlib.reload(asmpython_codegen) + if "ascii" not in reloaded_sema.BUILTINS: raise RuntimeError(f"ascii was not enabled in active sema module: {ASMPYTHON_SEMA}") + missing_exceptions = sorted( + set(_PORTAPY_EXCEPTION_PARENTS) - set(reloaded_sema.BUILTIN_EXCEPTIONS) + ) + missing_types = sorted( + set(_PORTAPY_TYPE_IDS) - set(reloaded_sema.BUILTIN_TYPE_NAMES) + ) + missing_codegen_exceptions = sorted( + set(_PORTAPY_EXCEPTION_IDS) - set(reloaded_codegen.BUILTIN_EXC_IDS) + ) + missing_codegen_types = sorted( + set(_PORTAPY_TYPE_IDS) - set(reloaded_codegen.BUILTIN_TYPE_IDS) + ) + if ( + missing_exceptions + or missing_types + or missing_codegen_exceptions + or missing_codegen_types + ): + raise RuntimeError( + "PortaPy compiler symbol enablement failed; " + f"sema_exceptions={missing_exceptions}, sema_types={missing_types}, " + f"codegen_exceptions={missing_codegen_exceptions}, " + f"codegen_types={missing_codegen_types}" + ) print("ASMPYTHON SEMA PATH", ASMPYTHON_SEMA) - print("ENABLED ASMPYTHON ASCII BUILTIN", reloaded.BUILTINS["ascii"]) + print("ASMPYTHON CODEGEN PATH", ASMPYTHON_CODEGEN) + print("ENABLED ASMPYTHON ASCII BUILTIN", reloaded_sema.BUILTINS["ascii"]) + print( + "ENABLED PORTAPY COMPILER SYMBOLS", + len(_PORTAPY_EXCEPTION_IDS), + len(_PORTAPY_TYPE_IDS), + ) def main() -> int: @@ -118,7 +256,7 @@ def main() -> int: if not path.is_file(): raise RuntimeError(f"missing asmpython stdlib source: {path}") _normalize_ascii_file(path, f"ASMPYTHON {name}") - _enable_compiler_ascii() + _enable_compiler_builtins() return 0 From 3a89576000ce4c92c010da5a7fc1e1c769d80f51 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:47:45 -0500 Subject: [PATCH 053/450] Compile production runtime through proven full-core CLI --- tools/build_native_full_runtime.py | 53 ++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tools/build_native_full_runtime.py b/tools/build_native_full_runtime.py index af666f31..c3f872e4 100644 --- a/tools/build_native_full_runtime.py +++ b/tools/build_native_full_runtime.py @@ -4,6 +4,8 @@ import argparse import json from pathlib import Path +import shlex +import subprocess import sys @@ -22,6 +24,7 @@ SOURCE = REPOSITORY_ROOT / "src" / "portapy" / "native_full_reference_entry.py" +COMPILER_WRAPPER = REPOSITORY_ROOT / "tools" / "run_full_core_asmpython.py" def _prepare_full_runtime_sources() -> None: @@ -29,6 +32,55 @@ def _prepare_full_runtime_sources() -> None: normalize_full_runtime() +def _install_full_runtime_compiler() -> None: + """Route production compilation through the proven full-core CLI wrapper.""" + + def compile_python_source( + *, + target: str, + source: Path, + output: Path, + build_log: Path, + ) -> Path: + command = [ + sys.executable, + str(COMPILER_WRAPPER), + "build", + str(source), + "--target", + target, + "--type", + "library", + "--backend", + "legacy", + "--no-pyinbin-fallback", + "--keep-assembly", + "-o", + str(output), + ] + completed = subprocess.run( + command, + cwd=REPOSITORY_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + build_log.parent.mkdir(parents=True, exist_ok=True) + build_log.write_text(completed.stdout, encoding="utf-8") + + assembly = output.with_suffix(".asm") + if not assembly.is_file(): + rendered = shlex.join(command) + raise BuildFailure( + f"asmpython did not emit assembly (exit {completed.returncode}): " + f"{rendered}\n{completed.stdout}" + ) + return assembly + + base_build._compile_python_source = compile_python_source + + def _install_full_runtime_transforms() -> None: original_transform = base_build._transform_assembly @@ -79,6 +131,7 @@ def build_full_runtime( ) -> dict[str, object]: if normalize: _prepare_full_runtime_sources() + _install_full_runtime_compiler() _install_full_runtime_transforms() metadata = base_build.build_native( From 824d4a2e9c5325b9d9e96875ea2fe50d9a5eb44d Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:48:02 -0500 Subject: [PATCH 054/450] Test pinned compiler symbol normalization --- ...st_normalize_full_core_compiler_symbols.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 tests/test_normalize_full_core_compiler_symbols.py diff --git a/tests/test_normalize_full_core_compiler_symbols.py b/tests/test_normalize_full_core_compiler_symbols.py new file mode 100644 index 00000000..ad22a194 --- /dev/null +++ b/tests/test_normalize_full_core_compiler_symbols.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from tools import normalize_full_core_lambdas as normalizer + + +def _sema_source() -> str: + return '''BUILTINS = { + "repr": (1, 1), +} +BUILTIN_EXCEPTIONS = frozenset({ + "FileNotFoundError", +}) +BUILTIN_TYPE_NAMES = frozenset({ + "int", "float", "str", "bool", "list", "dict", "tuple", "set", +}) +''' + + +def _codegen_source() -> str: + return '''BUILTIN_EXC_PARENTS = { + "StopIteration": "Exception", +} +BUILTIN_EXC_IDS = { + "StopIteration": 21, + "IOError": 19, # alias for OSError (same id) +} +BUILTIN_TYPE_IDS = { + "set": -8, +} +''' + + +def test_enables_all_portapy_compiler_symbols() -> None: + sema, codegen = normalizer._patch_compiler_runtime_symbols( + _sema_source(), _codegen_source() + ) + for name in normalizer._PORTAPY_EXCEPTION_IDS: + assert f' "{name}",' in sema + assert f' "{name}":' in codegen + for name, identifier in normalizer._PORTAPY_TYPE_IDS.items(): + assert f' "{name}",' in sema + assert f' "{name}": {identifier},' in codegen + + +def test_compiler_symbol_patch_is_idempotent() -> None: + sema, codegen = normalizer._patch_compiler_runtime_symbols( + _sema_source(), _codegen_source() + ) + assert normalizer._patch_compiler_runtime_symbols(sema, codegen) == ( + sema, + codegen, + ) From 439f7ee3ac7a646e44eb70abe901bea0fecb479e Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:50:54 -0500 Subject: [PATCH 055/450] Scope compiler symbol detection to target tables --- tools/normalize_full_core_lambdas.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tools/normalize_full_core_lambdas.py b/tools/normalize_full_core_lambdas.py index d862d10c..6936aa57 100644 --- a/tools/normalize_full_core_lambdas.py +++ b/tools/normalize_full_core_lambdas.py @@ -69,6 +69,16 @@ def _normalize_ascii_file(path: Path, label: str) -> None: print(f"REPLACED {label} ASCII CONVERSIONS", conversion_count) +def _target_table(source: str, marker: str, label: str) -> str: + if source.count(marker) != 1: + raise RuntimeError(f"{label}: expected one insertion marker") + marker_index = source.index(marker) + table_start = source.rfind("{", 0, marker_index) + if table_start < 0: + raise RuntimeError(f"{label}: table start not found") + return source[table_start:marker_index + len(marker)] + + def _insert_mapping_entries( source: str, *, @@ -76,11 +86,10 @@ def _insert_mapping_entries( entries: dict[str, object], label: str, ) -> str: - missing = [name for name in entries if f' "{name}":' not in source] + table = _target_table(source, marker, label) + missing = [name for name in entries if f' "{name}":' not in table] if not missing: return source - if source.count(marker) != 1: - raise RuntimeError(f"{label}: expected one insertion marker") lines = "".join(f' "{name}": {entries[name]!r},\n' for name in missing) return source.replace(marker, lines + marker, 1) @@ -92,11 +101,10 @@ def _insert_set_entries( names: tuple[str, ...], label: str, ) -> str: - missing = [name for name in names if f' "{name}",' not in source] + table = _target_table(source, marker, label) + missing = [name for name in names if f' "{name}",' not in table] if not missing: return source - if source.count(marker) != 1: - raise RuntimeError(f"{label}: expected one insertion marker") lines = "".join(f' "{name}",\n' for name in missing) return source.replace(marker, lines + marker, 1) From 9ebb5c44da8a57740c847f3324bb2cbdd7ca2c6d Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:51:17 -0500 Subject: [PATCH 056/450] Strengthen compiler symbol table tests --- tests/test_normalize_full_core_compiler_symbols.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_normalize_full_core_compiler_symbols.py b/tests/test_normalize_full_core_compiler_symbols.py index ad22a194..dbf7190a 100644 --- a/tests/test_normalize_full_core_compiler_symbols.py +++ b/tests/test_normalize_full_core_compiler_symbols.py @@ -6,6 +6,7 @@ def _sema_source() -> str: return '''BUILTINS = { "repr": (1, 1), + "type": (1, 1), } BUILTIN_EXCEPTIONS = frozenset({ "FileNotFoundError", @@ -34,9 +35,11 @@ def test_enables_all_portapy_compiler_symbols() -> None: sema, codegen = normalizer._patch_compiler_runtime_symbols( _sema_source(), _codegen_source() ) - for name in normalizer._PORTAPY_EXCEPTION_IDS: + for name, parent in normalizer._PORTAPY_EXCEPTION_PARENTS.items(): assert f' "{name}",' in sema - assert f' "{name}":' in codegen + assert f' "{name}": {parent!r},' in codegen + for name, identifier in normalizer._PORTAPY_EXCEPTION_IDS.items(): + assert f' "{name}": {identifier},' in codegen for name, identifier in normalizer._PORTAPY_TYPE_IDS.items(): assert f' "{name}",' in sema assert f' "{name}": {identifier},' in codegen From 8fba2edc58b72dda9189df3bb9c511b3ef5ced2f Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:52:01 -0500 Subject: [PATCH 057/450] Load asmpython lazily in compiler symbol normalizer --- tools/normalize_full_core_lambdas.py | 43 ++++++++++++++++------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/tools/normalize_full_core_lambdas.py b/tools/normalize_full_core_lambdas.py index 6936aa57..b0ec59d2 100644 --- a/tools/normalize_full_core_lambdas.py +++ b/tools/normalize_full_core_lambdas.py @@ -5,18 +5,10 @@ from pathlib import Path import re -import asmpython -import asmpython._compiler.codegen as asmpython_codegen -import asmpython._compiler.sema as asmpython_sema - VM_PATH = Path("src/portapy/core/vm.py") FRONTEND_PATH = Path("src/portapy/core/frontend.py") BYTECODE_PATH = Path("src/portapy/core/bytecode.py") -ASMPYTHON_ROOT = Path(asmpython.__file__).resolve().parent -ASMPYTHON_STDLIB = ASMPYTHON_ROOT / "stdlib" -ASMPYTHON_SEMA = Path(asmpython_sema.__file__).resolve() -ASMPYTHON_CODEGEN = Path(asmpython_codegen.__file__).resolve() _FSTRING_CONVERTER_ASSIGN = ( @@ -53,6 +45,16 @@ } +def _load_compiler_modules(): + """Import asmpython only for a native build, never during hosted test collection.""" + import asmpython + import asmpython._compiler.codegen as asmpython_codegen + import asmpython._compiler.sema as asmpython_sema + + root = Path(asmpython.__file__).resolve().parent + return asmpython_sema, asmpython_codegen, root / "stdlib" + + def _normalize_ascii(source: str) -> tuple[str, int, int]: call_count = source.count("ascii(") conversion_count = source.count("!a") @@ -146,8 +148,12 @@ def _patch_compiler_runtime_symbols( return sema_source, codegen_source -def _enable_compiler_builtins() -> None: - sema_source = ASMPYTHON_SEMA.read_text(encoding="utf-8") +def _enable_compiler_builtins() -> Path: + asmpython_sema, asmpython_codegen, stdlib = _load_compiler_modules() + sema_path = Path(asmpython_sema.__file__).resolve() + codegen_path = Path(asmpython_codegen.__file__).resolve() + + sema_source = sema_path.read_text(encoding="utf-8") ascii_marker = ' "repr": (1, 1),\n' if ascii_marker not in sema_source: raise RuntimeError("asmpython semantic builtin table is missing repr") @@ -158,19 +164,19 @@ def _enable_compiler_builtins() -> None: 1, ) - codegen_source = ASMPYTHON_CODEGEN.read_text(encoding="utf-8") + codegen_source = codegen_path.read_text(encoding="utf-8") sema_source, codegen_source = _patch_compiler_runtime_symbols( sema_source, codegen_source, ) - ASMPYTHON_SEMA.write_text(sema_source, encoding="utf-8") - ASMPYTHON_CODEGEN.write_text(codegen_source, encoding="utf-8") + sema_path.write_text(sema_source, encoding="utf-8") + codegen_path.write_text(codegen_source, encoding="utf-8") importlib.invalidate_caches() reloaded_sema = importlib.reload(asmpython_sema) reloaded_codegen = importlib.reload(asmpython_codegen) if "ascii" not in reloaded_sema.BUILTINS: - raise RuntimeError(f"ascii was not enabled in active sema module: {ASMPYTHON_SEMA}") + raise RuntimeError(f"ascii was not enabled in active sema module: {sema_path}") missing_exceptions = sorted( set(_PORTAPY_EXCEPTION_PARENTS) - set(reloaded_sema.BUILTIN_EXCEPTIONS) ) @@ -195,14 +201,15 @@ def _enable_compiler_builtins() -> None: f"codegen_exceptions={missing_codegen_exceptions}, " f"codegen_types={missing_codegen_types}" ) - print("ASMPYTHON SEMA PATH", ASMPYTHON_SEMA) - print("ASMPYTHON CODEGEN PATH", ASMPYTHON_CODEGEN) + print("ASMPYTHON SEMA PATH", sema_path) + print("ASMPYTHON CODEGEN PATH", codegen_path) print("ENABLED ASMPYTHON ASCII BUILTIN", reloaded_sema.BUILTINS["ascii"]) print( "ENABLED PORTAPY COMPILER SYMBOLS", len(_PORTAPY_EXCEPTION_IDS), len(_PORTAPY_TYPE_IDS), ) + return stdlib def main() -> int: @@ -259,12 +266,12 @@ def main() -> int: print("REPLACED FRONTEND ASCII CONVERSIONS", frontend_ascii_conversion_count) _normalize_ascii_file(BYTECODE_PATH, "BYTECODE") + stdlib = _enable_compiler_builtins() for name in ("dataclasses.py", "enum.py", "types.py"): - path = ASMPYTHON_STDLIB / name + path = stdlib / name if not path.is_file(): raise RuntimeError(f"missing asmpython stdlib source: {path}") _normalize_ascii_file(path, f"ASMPYTHON {name}") - _enable_compiler_builtins() return 0 From 66a6db75101edd1ba94f949afb9a878f78fcd2e5 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:56:05 -0500 Subject: [PATCH 058/450] Use native-safe error formatting in full Runtime ABI --- tools/normalize_full_reference_abi_helpers.py | 60 ++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/tools/normalize_full_reference_abi_helpers.py b/tools/normalize_full_reference_abi_helpers.py index 2b2ed84a..c35875d0 100644 --- a/tools/normalize_full_reference_abi_helpers.py +++ b/tools/normalize_full_reference_abi_helpers.py @@ -106,7 +106,32 @@ def _is_utf8_source_upper_bound(node: ast.AST) -> bool: ) +def _is_traceback_format_join(node: ast.AST) -> bool: + """Return True for ``"".join(traceback.format_exception(error))``.""" + if not isinstance(node, ast.Call) or len(node.args) != 1 or node.keywords: + return False + if not isinstance(node.func, ast.Attribute) or node.func.attr != "join": + return False + if not isinstance(node.func.value, ast.Constant) or node.func.value.value != "": + return False + inner = node.args[0] + return ( + isinstance(inner, ast.Call) + and len(inner.args) == 1 + and not inner.keywords + and isinstance(inner.func, ast.Attribute) + and inner.func.attr == "format_exception" + and isinstance(inner.func.value, ast.Name) + and inner.func.value.id == "traceback" + and isinstance(inner.args[0], ast.Name) + and inner.args[0].id == "error" + ) + + class _Rewrite(ast.NodeTransformer): + def __init__(self) -> None: + self.traceback_rewrites = 0 + def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST | None: if node.name in _DROP: return None @@ -143,10 +168,29 @@ def visit_BoolOp(self, node: ast.BoolOp) -> ast.AST: return ast.copy_location(values[0], node) return ast.copy_location(ast.BoolOp(op=node.op, values=values), node) + def visit_Call(self, node: ast.Call) -> ast.AST: + if _is_traceback_format_join(node): + self.traceback_rewrites += 1 + return ast.copy_location( + ast.Call( + func=ast.Name(id="str", ctx=ast.Load()), + args=[ast.Name(id="error", ctx=ast.Load())], + keywords=[], + ), + node, + ) + return self.generic_visit(node) + def main() -> int: module = ast.parse(PATH.read_text(encoding="utf-8")) - module = _Rewrite().visit(module) + rewrite = _Rewrite() + module = rewrite.visit(module) + if rewrite.traceback_rewrites != 1: + raise RuntimeError( + "full Runtime traceback normalization failed: " + f"expected 1 call, found {rewrite.traceback_rewrites}" + ) if any( isinstance(node, ast.ClassDef) and node.name == "_PortaPyImportLoader" for node in module.body @@ -175,6 +219,9 @@ def main() -> int: for node in ast.walk(verified) if _is_utf8_source_upper_bound(node) ] + stale_tracebacks = [ + node for node in ast.walk(verified) if _is_traceback_format_join(node) + ] runtime_create = next( ( node @@ -190,11 +237,19 @@ def main() -> int: and "__pyinbin_import__" in runtime_create_text ) builtins_ready = "_seed_builtins" in runtime_create_text - if missing or stale or unsafe_spans or not loader_ready or not builtins_ready: + if ( + missing + or stale + or unsafe_spans + or stale_tracebacks + or not loader_ready + or not builtins_ready + ): raise RuntimeError( "full Runtime ABI helper normalization failed; " f"missing={missing}, stale={stale}, " f"unsafe_utf8_spans={len(unsafe_spans)}, " + f"stale_tracebacks={len(stale_tracebacks)}, " f"loader_ready={loader_ready}, builtins_ready={builtins_ready}" ) print( @@ -203,6 +258,7 @@ def main() -> int: len(_DROP), "BUILTINS", "IMPORT_LOADER", + "TRACEBACK", ) return 0 From 95cbef8ad7189f5feec3fcdf96f3e99a95e9a5ef Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:56:36 -0500 Subject: [PATCH 059/450] Test native-safe full Runtime error formatting --- tests/test_full_reference_abi_normalization.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_full_reference_abi_normalization.py b/tests/test_full_reference_abi_normalization.py index 6c65a9b6..c06cca21 100644 --- a/tests/test_full_reference_abi_normalization.py +++ b/tests/test_full_reference_abi_normalization.py @@ -27,6 +27,15 @@ def test_full_reference_normalization_installs_runtime_support( } assert "_PortaPyImportLoader" in classes assert "source_size > len(source)" not in source + assert "traceback.format_exception" not in source + + capture = next( + node + for node in ast.walk(module) + if isinstance(node, ast.FunctionDef) and node.name == "_capture" + ) + capture_source = ast.unparse(capture) + assert "str(error)" in capture_source runtime_create = next( node From 6afdb92104e2e1c820652b97bbe9fc396de45bc5 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:56:37 -0500 Subject: [PATCH 060/450] Normalize native reference error capture --- tools/normalize_full_reference_runtime.py | 33 +++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 tools/normalize_full_reference_runtime.py diff --git a/tools/normalize_full_reference_runtime.py b/tools/normalize_full_reference_runtime.py new file mode 100644 index 00000000..f6ae47b2 --- /dev/null +++ b/tools/normalize_full_reference_runtime.py @@ -0,0 +1,33 @@ +"""Remove host-only traceback formatting from the native reference runtime.""" +from __future__ import annotations + +from pathlib import Path + + +PATH = Path("src/portapy/reference_api.py") + +_TRACEBACK_IMPORT = "import traceback\n" +_TRACEBACK_FORMAT = ' "".join(traceback.format_exception(error)),\n' +_NATIVE_FORMAT = ( + ' type(error).__name__ + ": " + str(error),\n' +) + + +def main() -> int: + source = PATH.read_text(encoding="utf-8") + import_count = source.count(_TRACEBACK_IMPORT) + format_count = source.count(_TRACEBACK_FORMAT) + if import_count != 1 or format_count != 1: + raise RuntimeError( + "native reference traceback normalization expected one import and " + f"one formatter; imports={import_count}, formatters={format_count}" + ) + source = source.replace(_TRACEBACK_IMPORT, "", 1) + source = source.replace(_TRACEBACK_FORMAT, _NATIVE_FORMAT, 1) + PATH.write_text(source, encoding="utf-8") + print("NORMALIZED NATIVE REFERENCE ERROR CAPTURE", format_count) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 4d6f8a01f26216420ed58c7824bf98326a14b749 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:57:06 -0500 Subject: [PATCH 061/450] Include native reference error normalization --- tools/normalize_full_core_validation.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/normalize_full_core_validation.py b/tools/normalize_full_core_validation.py index 6160880a..1b71a145 100644 --- a/tools/normalize_full_core_validation.py +++ b/tools/normalize_full_core_validation.py @@ -26,6 +26,9 @@ from tools.normalize_full_reference_abi_helpers import ( main as normalize_reference_abi_helpers, ) +from tools.normalize_full_reference_runtime import ( + main as normalize_reference_runtime, +) BYTECODE_PATH = Path("src/portapy/core/bytecode.py") @@ -64,13 +67,12 @@ def _normalize_opcode_validation() -> None: def main() -> int: - # These four passes were historically run only by the probe workflow. - # Keeping them here makes clean production builds use the same proven - # compiler-safe source preparation as the full-core transition probe. + # These passes prepare the source modules imported by the generated entry. normalize_probe() normalize_lambdas() normalize_native_semantics() normalize_opcode_maps() + normalize_reference_runtime() materialize_reference_entry() normalize_reference_abi_helpers() From 47cb18b6521ce2fa2d9babd67f8ab890e462bc49 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:57:18 -0500 Subject: [PATCH 062/450] Test native reference error normalization --- .../test_normalize_full_reference_runtime.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/test_normalize_full_reference_runtime.py diff --git a/tests/test_normalize_full_reference_runtime.py b/tests/test_normalize_full_reference_runtime.py new file mode 100644 index 00000000..ae8dafc7 --- /dev/null +++ b/tests/test_normalize_full_reference_runtime.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tools import normalize_full_reference_runtime as normalizer + + +def test_replaces_host_traceback_formatter( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "reference_api.py" + path.write_text( + "import traceback\n" + "def capture(error):\n" + " return ErrorInfo(\n" + ' "".join(traceback.format_exception(error)),\n' + " )\n", + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", path) + assert normalizer.main() == 0 + result = path.read_text(encoding="utf-8") + assert "import traceback" not in result + assert "format_exception" not in result + assert 'type(error).__name__ + ": " + str(error)' in result + + +def test_rejects_unexpected_traceback_shape( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "reference_api.py" + path.write_text("import traceback\n", encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + with pytest.raises(RuntimeError, match="one import and one formatter"): + normalizer.main() From fe4f8998f854df9ddc97763838a3960db8be3a8a Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:00:24 -0500 Subject: [PATCH 063/450] Keep traceback normalization on reference source --- tools/normalize_full_reference_abi_helpers.py | 60 +------------------ 1 file changed, 2 insertions(+), 58 deletions(-) diff --git a/tools/normalize_full_reference_abi_helpers.py b/tools/normalize_full_reference_abi_helpers.py index c35875d0..2b2ed84a 100644 --- a/tools/normalize_full_reference_abi_helpers.py +++ b/tools/normalize_full_reference_abi_helpers.py @@ -106,32 +106,7 @@ def _is_utf8_source_upper_bound(node: ast.AST) -> bool: ) -def _is_traceback_format_join(node: ast.AST) -> bool: - """Return True for ``"".join(traceback.format_exception(error))``.""" - if not isinstance(node, ast.Call) or len(node.args) != 1 or node.keywords: - return False - if not isinstance(node.func, ast.Attribute) or node.func.attr != "join": - return False - if not isinstance(node.func.value, ast.Constant) or node.func.value.value != "": - return False - inner = node.args[0] - return ( - isinstance(inner, ast.Call) - and len(inner.args) == 1 - and not inner.keywords - and isinstance(inner.func, ast.Attribute) - and inner.func.attr == "format_exception" - and isinstance(inner.func.value, ast.Name) - and inner.func.value.id == "traceback" - and isinstance(inner.args[0], ast.Name) - and inner.args[0].id == "error" - ) - - class _Rewrite(ast.NodeTransformer): - def __init__(self) -> None: - self.traceback_rewrites = 0 - def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST | None: if node.name in _DROP: return None @@ -168,29 +143,10 @@ def visit_BoolOp(self, node: ast.BoolOp) -> ast.AST: return ast.copy_location(values[0], node) return ast.copy_location(ast.BoolOp(op=node.op, values=values), node) - def visit_Call(self, node: ast.Call) -> ast.AST: - if _is_traceback_format_join(node): - self.traceback_rewrites += 1 - return ast.copy_location( - ast.Call( - func=ast.Name(id="str", ctx=ast.Load()), - args=[ast.Name(id="error", ctx=ast.Load())], - keywords=[], - ), - node, - ) - return self.generic_visit(node) - def main() -> int: module = ast.parse(PATH.read_text(encoding="utf-8")) - rewrite = _Rewrite() - module = rewrite.visit(module) - if rewrite.traceback_rewrites != 1: - raise RuntimeError( - "full Runtime traceback normalization failed: " - f"expected 1 call, found {rewrite.traceback_rewrites}" - ) + module = _Rewrite().visit(module) if any( isinstance(node, ast.ClassDef) and node.name == "_PortaPyImportLoader" for node in module.body @@ -219,9 +175,6 @@ def main() -> int: for node in ast.walk(verified) if _is_utf8_source_upper_bound(node) ] - stale_tracebacks = [ - node for node in ast.walk(verified) if _is_traceback_format_join(node) - ] runtime_create = next( ( node @@ -237,19 +190,11 @@ def main() -> int: and "__pyinbin_import__" in runtime_create_text ) builtins_ready = "_seed_builtins" in runtime_create_text - if ( - missing - or stale - or unsafe_spans - or stale_tracebacks - or not loader_ready - or not builtins_ready - ): + if missing or stale or unsafe_spans or not loader_ready or not builtins_ready: raise RuntimeError( "full Runtime ABI helper normalization failed; " f"missing={missing}, stale={stale}, " f"unsafe_utf8_spans={len(unsafe_spans)}, " - f"stale_tracebacks={len(stale_tracebacks)}, " f"loader_ready={loader_ready}, builtins_ready={builtins_ready}" ) print( @@ -258,7 +203,6 @@ def main() -> int: len(_DROP), "BUILTINS", "IMPORT_LOADER", - "TRACEBACK", ) return 0 From 34bab80f018e4296ab2ed4be6a881df9d2f6147d Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:00:35 -0500 Subject: [PATCH 064/450] Test ABI helper normalization independently --- tests/test_full_reference_abi_normalization.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/test_full_reference_abi_normalization.py b/tests/test_full_reference_abi_normalization.py index c06cca21..6c65a9b6 100644 --- a/tests/test_full_reference_abi_normalization.py +++ b/tests/test_full_reference_abi_normalization.py @@ -27,15 +27,6 @@ def test_full_reference_normalization_installs_runtime_support( } assert "_PortaPyImportLoader" in classes assert "source_size > len(source)" not in source - assert "traceback.format_exception" not in source - - capture = next( - node - for node in ast.walk(module) - if isinstance(node, ast.FunctionDef) and node.name == "_capture" - ) - capture_source = ast.unparse(capture) - assert "str(error)" in capture_source runtime_create = next( node From c9f09677de073a31f9c73d77a73d81db70258a67 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:06:08 -0500 Subject: [PATCH 065/450] Link Linux standalone runtime against libm --- tools/build_native.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/build_native.py b/tools/build_native.py index cb248333..ad0afc8b 100644 --- a/tools/build_native.py +++ b/tools/build_native.py @@ -263,6 +263,7 @@ def build_native( "-shared", *link_objects, f"-Wl,--version-script={version_script}", + "-lm", "-o", str(output), ] From fb961f12198a24893ece236ace68d41a0a5305af Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:06:40 -0500 Subject: [PATCH 066/450] Link the standalone Linux runtime with libm --- tools/build_native_host_calls.py | 38 +++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/tools/build_native_host_calls.py b/tools/build_native_host_calls.py index 82f94a39..7d8c4c04 100644 --- a/tools/build_native_host_calls.py +++ b/tools/build_native_host_calls.py @@ -76,6 +76,30 @@ def _compile_bridge_glue( _run(command, log=log) +def _linux_link_command( + *, + gcc: str, + objects: list[str], + version_script: Path, + output: Path, +) -> list[str]: + """Return a self-contained ELF shared-library link command. + + asmpython emits calls to ``pow`` for Python exponentiation. ``libportapy.so`` + must therefore declare its own libm dependency rather than relying on the + embedding process to have loaded libm globally already. + """ + return [ + gcc, + "-shared", + *objects, + f"-Wl,--version-script={version_script}", + "-lm", + "-o", + str(output), + ] + + def _upgrade_linked_artifact( *, target: str, @@ -179,14 +203,12 @@ def _upgrade_linked_artifact( linux_version_script(host_bridge=True, host_calls=True), encoding="utf-8", ) - command = [ - gcc, - "-shared", - *objects, - f"-Wl,--version-script={version_script}", - "-o", - str(output), - ] + command = _linux_link_command( + gcc=gcc, + objects=objects, + version_script=version_script, + output=output, + ) else: definition = work_dir / "portapy-host-calls.def" definition.write_text( From 605f5e3ee2e309cf06969d6b4e7d8b50a3d650bb Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:06:59 -0500 Subject: [PATCH 067/450] Test standalone Linux math linkage --- tests/test_build_native_host_calls.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/test_build_native_host_calls.py diff --git a/tests/test_build_native_host_calls.py b/tests/test_build_native_host_calls.py new file mode 100644 index 00000000..c95ccd83 --- /dev/null +++ b/tests/test_build_native_host_calls.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from pathlib import Path + +from tools.build_native_host_calls import _linux_link_command + + +def test_linux_shared_library_links_its_math_dependency() -> None: + command = _linux_link_command( + gcc="gcc", + objects=["runtime.o", "glue.o"], + version_script=Path("portapy.map"), + output=Path("libportapy.so"), + ) + + assert command == [ + "gcc", + "-shared", + "runtime.o", + "glue.o", + "-Wl,--version-script=portapy.map", + "-lm", + "-o", + "libportapy.so", + ] From 625b5e7fd1653b9b589568aae1d9e159bc1283d8 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:09:45 -0500 Subject: [PATCH 068/450] Add focused Windows native build diagnostics --- .github/workflows/native-build-diagnostic.yml | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/.github/workflows/native-build-diagnostic.yml b/.github/workflows/native-build-diagnostic.yml index 41cae7b1..0b512faa 100644 --- a/.github/workflows/native-build-diagnostic.yml +++ b/.github/workflows/native-build-diagnostic.yml @@ -85,3 +85,81 @@ jobs: run: | printf '%s\n' "$BUILD_REPORT" exit "$BUILD_STATUS" + + build-windows: + runs-on: windows-2022 + outputs: + status: ${{ steps.build.outputs.status }} + report: ${{ steps.build.outputs.report }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install quietly + shell: powershell + run: | + $installLog = Join-Path $env:RUNNER_TEMP 'portapy-install-windows.log' + choco install nasm mingw -y --no-progress *> $installLog + if ($LASTEXITCODE -ne 0) { Get-Content $installLog; exit $LASTEXITCODE } + python -m pip install -e '.[test]' *>> $installLog + if ($LASTEXITCODE -ne 0) { Get-Content $installLog; exit $LASTEXITCODE } + python -m pip install --no-cache-dir --force-reinstall ` + 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' ` + *>> $installLog + if ($LASTEXITCODE -ne 0) { Get-Content $installLog; exit $LASTEXITCODE } + - name: Capture compiler diagnostic + id: build + shell: powershell + run: | + $buildLog = Join-Path $env:RUNNER_TEMP 'portapy-build-windows.log' + $reportLog = Join-Path $env:RUNNER_TEMP 'portapy-report-windows.log' + $output = Join-Path $env:RUNNER_TEMP 'portapy.dll' + $workDir = Join-Path $env:RUNNER_TEMP 'portapy-build-windows' + python tools/build_native_typed.py ` + --target windows ` + --output $output ` + --work-dir $workDir ` + *> $buildLog + $status = $LASTEXITCODE + $lines = Get-Content $buildLog + $errorLines = $lines | Select-String -Pattern 'error|failed|failure|unsupported|traceback|exception|undefined|invalid' + @( + '=== FIRST 35 LINES ===' + $lines | Select-Object -First 35 + '' + '=== ERROR-LIKE LINES ===' + $errorLines | Select-Object -Last 120 + '' + '=== FINAL 120 LINES ===' + $lines | Select-Object -Last 120 + ) | Set-Content -Encoding utf8 $reportLog + "status=$status" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + 'report< Date: Tue, 21 Jul 2026 16:12:16 -0500 Subject: [PATCH 069/450] Use string keys for native value handles --- tools/normalize_full_reference_runtime.py | 81 ++++++++++++++++++++--- 1 file changed, 70 insertions(+), 11 deletions(-) diff --git a/tools/normalize_full_reference_runtime.py b/tools/normalize_full_reference_runtime.py index f6ae47b2..5e84ab85 100644 --- a/tools/normalize_full_reference_runtime.py +++ b/tools/normalize_full_reference_runtime.py @@ -1,4 +1,4 @@ -"""Remove host-only traceback formatting from the native reference runtime.""" +"""Normalize host-only features in the native reference runtime.""" from __future__ import annotations from pathlib import Path @@ -11,21 +11,80 @@ _NATIVE_FORMAT = ( ' type(error).__name__ + ": " + str(error),\n' ) +_VALUES_ANNOTATION = "self._values: dict[int, _Slot] = {}" +_NATIVE_VALUES_ANNOTATION = "self._values: dict[str, _Slot] = {}" +_HANDLE_SUBSCRIPT = "self._values[handle]" +_NATIVE_HANDLE_SUBSCRIPT = "self._values[str(handle)]" +_HANDLE_GET = "self._values.get(handle)" +_NATIVE_HANDLE_GET = "self._values.get(str(handle))" +_CALLABLE_HANDLE_GET = "self._values.get(callable_handle)" +_NATIVE_CALLABLE_HANDLE_GET = "self._values.get(str(callable_handle))" -def main() -> int: - source = PATH.read_text(encoding="utf-8") - import_count = source.count(_TRACEBACK_IMPORT) - format_count = source.count(_TRACEBACK_FORMAT) - if import_count != 1 or format_count != 1: +def _replace_exact( + source: str, + old: str, + new: str, + *, + label: str, + expected: int, +) -> str: + count = source.count(old) + if count != expected: raise RuntimeError( - "native reference traceback normalization expected one import and " - f"one formatter; imports={import_count}, formatters={format_count}" + f"native reference {label} normalization expected {expected} matches, " + f"found {count}" ) - source = source.replace(_TRACEBACK_IMPORT, "", 1) - source = source.replace(_TRACEBACK_FORMAT, _NATIVE_FORMAT, 1) + return source.replace(old, new) + + +def main() -> int: + source = PATH.read_text(encoding="utf-8") + source = _replace_exact( + source, + _TRACEBACK_IMPORT, + "", + label="traceback import", + expected=1, + ) + source = _replace_exact( + source, + _TRACEBACK_FORMAT, + _NATIVE_FORMAT, + label="traceback formatter", + expected=1, + ) + source = _replace_exact( + source, + _VALUES_ANNOTATION, + _NATIVE_VALUES_ANNOTATION, + label="value-table annotation", + expected=1, + ) + source = _replace_exact( + source, + _HANDLE_SUBSCRIPT, + _NATIVE_HANDLE_SUBSCRIPT, + label="value-table subscript", + expected=2, + ) + source = _replace_exact( + source, + _HANDLE_GET, + _NATIVE_HANDLE_GET, + label="value-table lookup", + expected=8, + ) + source = _replace_exact( + source, + _CALLABLE_HANDLE_GET, + _NATIVE_CALLABLE_HANDLE_GET, + label="callable value-table lookup", + expected=1, + ) PATH.write_text(source, encoding="utf-8") - print("NORMALIZED NATIVE REFERENCE ERROR CAPTURE", format_count) + print("NORMALIZED NATIVE REFERENCE ERROR CAPTURE", 1) + print("NORMALIZED NATIVE VALUE HANDLE KEYS", 11) return 0 From ac710bdfd71765f8663f85c55c96407f3335cb93 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:12:39 -0500 Subject: [PATCH 070/450] Test native string-keyed handle tables --- .../test_normalize_full_reference_runtime.py | 48 ++++++++++++++----- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/tests/test_normalize_full_reference_runtime.py b/tests/test_normalize_full_reference_runtime.py index ae8dafc7..9fc37562 100644 --- a/tests/test_normalize_full_reference_runtime.py +++ b/tests/test_normalize_full_reference_runtime.py @@ -7,31 +7,57 @@ from tools import normalize_full_reference_runtime as normalizer -def test_replaces_host_traceback_formatter( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - path = tmp_path / "reference_api.py" - path.write_text( +def _reference_source() -> str: + return ( "import traceback\n" - "def capture(error):\n" - " return ErrorInfo(\n" + "class Runtime:\n" + " def __init__(self):\n" + " self._values: dict[int, _Slot] = {}\n" + " def capture(self, error):\n" + " return ErrorInfo(\n" ' "".join(traceback.format_exception(error)),\n' - " )\n", - encoding="utf-8", + " )\n" + " def store(self, handle, value):\n" + " self._values[handle] = value\n" + " def remove(self, handle):\n" + " del self._values[handle]\n" + " def lookup1(self, handle): return self._values.get(handle)\n" + " def lookup2(self, handle): return self._values.get(handle)\n" + " def lookup3(self, handle): return self._values.get(handle)\n" + " def lookup4(self, handle): return self._values.get(handle)\n" + " def lookup5(self, handle): return self._values.get(handle)\n" + " def lookup6(self, handle): return self._values.get(handle)\n" + " def lookup7(self, handle): return self._values.get(handle)\n" + " def lookup8(self, handle): return self._values.get(handle)\n" + " def callable(self, callable_handle):\n" + " return self._values.get(callable_handle)\n" ) + + +def test_replaces_host_tracebacks_and_integer_handle_keys( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "reference_api.py" + path.write_text(_reference_source(), encoding="utf-8") monkeypatch.setattr(normalizer, "PATH", path) + assert normalizer.main() == 0 + result = path.read_text(encoding="utf-8") assert "import traceback" not in result assert "format_exception" not in result assert 'type(error).__name__ + ": " + str(error)' in result + assert "dict[str, _Slot]" in result + assert result.count("self._values[str(handle)]") == 2 + assert result.count("self._values.get(str(handle))") == 8 + assert result.count("self._values.get(str(callable_handle))") == 1 -def test_rejects_unexpected_traceback_shape( +def test_rejects_unexpected_reference_runtime_shape( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: path = tmp_path / "reference_api.py" path.write_text("import traceback\n", encoding="utf-8") monkeypatch.setattr(normalizer, "PATH", path) - with pytest.raises(RuntimeError, match="one import and one formatter"): + with pytest.raises(RuntimeError, match="traceback formatter normalization"): normalizer.main() From 60a463c123a37ef64e9ed18f321533b90ed9b6ef Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:21:59 -0500 Subject: [PATCH 071/450] Track native value kinds on handles --- tools/normalize_full_reference_runtime.py | 180 +++++++++++++++++++--- 1 file changed, 160 insertions(+), 20 deletions(-) diff --git a/tools/normalize_full_reference_runtime.py b/tools/normalize_full_reference_runtime.py index 5e84ab85..349de7bc 100644 --- a/tools/normalize_full_reference_runtime.py +++ b/tools/normalize_full_reference_runtime.py @@ -11,6 +11,152 @@ _NATIVE_FORMAT = ( ' type(error).__name__ + ": " + str(error),\n' ) +_SLOT = '''@dataclass +class _Slot: + value: object + refs: int = 1 +''' +_NATIVE_SLOT = '''@dataclass +class _Slot: + value: object + kind: ValueKind = ValueKind.INT + refs: int = 1 +''' +_STORE = ''' def _store(self, value: object) -> int: + handle = self._next + self._next += 1 + self._values[handle] = _Slot(value) + return handle +''' +_NATIVE_STORE = ''' def _store( + self, + value: object, + kind: ValueKind = ValueKind.INT, + ) -> int: + handle = self._next + self._next += 1 + self._values[handle] = _Slot(value, kind) + return handle +''' +_BOX_METHODS = ''' def box_none(self) -> tuple[Status, int]: + return Status.OK, self._store(None) + + def box_bool(self, value: bool) -> tuple[Status, int]: + return Status.OK, self._store(value) + + def box_int(self, value: int) -> tuple[Status, int]: + return Status.OK, self._store(value) + + def box_float(self, value: float) -> tuple[Status, int]: + return Status.OK, self._store(value) + + def box_utf8(self, value: str) -> tuple[Status, int]: + return Status.OK, self._store(value) + + def box_bytes(self, value: bytes) -> tuple[Status, int]: + return Status.OK, self._store(value) +''' +_NATIVE_BOX_METHODS = ''' def box_none(self) -> tuple[Status, int]: + return Status.OK, self._store(None, ValueKind.NONE) + + def box_bool(self, value: bool) -> tuple[Status, int]: + return Status.OK, self._store(value, ValueKind.BOOL) + + def box_int(self, value: int) -> tuple[Status, int]: + return Status.OK, self._store(value, ValueKind.INT) + + def box_float(self, value: float) -> tuple[Status, int]: + return Status.OK, self._store(value, ValueKind.FLOAT) + + def box_utf8(self, value: str) -> tuple[Status, int]: + return Status.OK, self._store(value, ValueKind.STRING) + + def box_bytes(self, value: bytes) -> tuple[Status, int]: + return Status.OK, self._store(value, ValueKind.BYTES) +''' +_VALUE_KIND_METHOD = ''' def value_kind(self, handle: int) -> tuple[Status, ValueKind]: + slot = self._values.get(handle) + if slot is None: + return self._capture(Status.INVALID_HANDLE, KeyError(handle)), ValueKind.OBJECT + value = slot.value + if value is None: + kind = ValueKind.NONE + elif type(value) is bool: + kind = ValueKind.BOOL + elif type(value) is int: + kind = ValueKind.INT + elif type(value) is float: + kind = ValueKind.FLOAT + elif type(value) is str: + kind = ValueKind.STRING + elif type(value) is bytes: + kind = ValueKind.BYTES + elif type(value) is tuple: + kind = ValueKind.TUPLE + elif type(value) is dict: + kind = ValueKind.DICT + elif type(value) is list: + kind = ValueKind.LIST + elif callable(value): + kind = ValueKind.CALLABLE + else: + kind = ValueKind.OBJECT + return Status.OK, kind +''' +_NATIVE_VALUE_KIND_METHOD = ''' def value_kind(self, handle: int) -> tuple[Status, ValueKind]: + slot = self._values.get(handle) + if slot is None: + return self._capture(Status.INVALID_HANDLE, KeyError(handle)), ValueKind.OBJECT + return Status.OK, slot.kind +''' +_AS_INT_METHOD = ''' def as_int(self, handle: int) -> tuple[Status, int]: + slot = self._values.get(handle) + if slot is None: + return self._capture(Status.INVALID_HANDLE, KeyError(handle)), 0 + if type(slot.value) is not int: + return self._capture(Status.TYPE_ERROR, TypeError("value is not int")), 0 + return Status.OK, slot.value +''' +_NATIVE_AS_INT_METHOD = ''' def as_int(self, handle: int) -> tuple[Status, int]: + slot = self._values.get(handle) + if slot is None: + return self._capture(Status.INVALID_HANDLE, KeyError(handle)), 0 + if slot.kind is not ValueKind.INT: + return self._capture(Status.TYPE_ERROR, TypeError("value is not int")), 0 + return Status.OK, slot.value +''' +_AS_FLOAT_METHOD = ''' def as_float(self, handle: int) -> tuple[Status, float]: + slot = self._values.get(handle) + if slot is None: + return self._capture(Status.INVALID_HANDLE, KeyError(handle)), 0.0 + if type(slot.value) is not float: + return self._capture(Status.TYPE_ERROR, TypeError("value is not float")), 0.0 + return Status.OK, slot.value +''' +_NATIVE_AS_FLOAT_METHOD = ''' def as_float(self, handle: int) -> tuple[Status, float]: + slot = self._values.get(handle) + if slot is None: + return self._capture(Status.INVALID_HANDLE, KeyError(handle)), 0.0 + if slot.kind is not ValueKind.FLOAT: + return self._capture(Status.TYPE_ERROR, TypeError("value is not float")), 0.0 + return Status.OK, slot.value +''' +_AS_UTF8_METHOD = ''' def as_utf8(self, handle: int) -> tuple[Status, bytes]: + slot = self._values.get(handle) + if slot is None: + return self._capture(Status.INVALID_HANDLE, KeyError(handle)), b"" + if type(slot.value) is not str: + return self._capture(Status.TYPE_ERROR, TypeError("value is not str")), b"" + return Status.OK, slot.value.encode("utf-8") +''' +_NATIVE_AS_UTF8_METHOD = ''' def as_utf8(self, handle: int) -> tuple[Status, bytes]: + slot = self._values.get(handle) + if slot is None: + return self._capture(Status.INVALID_HANDLE, KeyError(handle)), b"" + if slot.kind is not ValueKind.STRING: + return self._capture(Status.TYPE_ERROR, TypeError("value is not str")), b"" + return Status.OK, slot.value.encode("utf-8") +''' _VALUES_ANNOTATION = "self._values: dict[int, _Slot] = {}" _NATIVE_VALUES_ANNOTATION = "self._values: dict[str, _Slot] = {}" _HANDLE_SUBSCRIPT = "self._values[handle]" @@ -40,27 +186,20 @@ def _replace_exact( def main() -> int: source = PATH.read_text(encoding="utf-8") - source = _replace_exact( - source, - _TRACEBACK_IMPORT, - "", - label="traceback import", - expected=1, - ) - source = _replace_exact( - source, - _TRACEBACK_FORMAT, - _NATIVE_FORMAT, - label="traceback formatter", - expected=1, - ) - source = _replace_exact( - source, - _VALUES_ANNOTATION, - _NATIVE_VALUES_ANNOTATION, - label="value-table annotation", - expected=1, + replacements = ( + (_TRACEBACK_IMPORT, "", "traceback import"), + (_TRACEBACK_FORMAT, _NATIVE_FORMAT, "traceback formatter"), + (_SLOT, _NATIVE_SLOT, "value slot"), + (_STORE, _NATIVE_STORE, "value store"), + (_BOX_METHODS, _NATIVE_BOX_METHODS, "scalar boxing"), + (_VALUE_KIND_METHOD, _NATIVE_VALUE_KIND_METHOD, "value kind"), + (_AS_INT_METHOD, _NATIVE_AS_INT_METHOD, "integer conversion"), + (_AS_FLOAT_METHOD, _NATIVE_AS_FLOAT_METHOD, "float conversion"), + (_AS_UTF8_METHOD, _NATIVE_AS_UTF8_METHOD, "UTF-8 conversion"), + (_VALUES_ANNOTATION, _NATIVE_VALUES_ANNOTATION, "value-table annotation"), ) + for old, new, label in replacements: + source = _replace_exact(source, old, new, label=label, expected=1) source = _replace_exact( source, _HANDLE_SUBSCRIPT, @@ -85,6 +224,7 @@ def main() -> int: PATH.write_text(source, encoding="utf-8") print("NORMALIZED NATIVE REFERENCE ERROR CAPTURE", 1) print("NORMALIZED NATIVE VALUE HANDLE KEYS", 11) + print("NORMALIZED NATIVE VALUE KIND SLOTS", 9) return 0 From 17da49d83f41a3067781fa886e119b6ebb4d8e49 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:23:22 -0500 Subject: [PATCH 072/450] Test native value kind normalization --- .../test_normalize_full_reference_runtime.py | 52 ++++++++----------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/tests/test_normalize_full_reference_runtime.py b/tests/test_normalize_full_reference_runtime.py index 9fc37562..1ed47f84 100644 --- a/tests/test_normalize_full_reference_runtime.py +++ b/tests/test_normalize_full_reference_runtime.py @@ -7,38 +7,14 @@ from tools import normalize_full_reference_runtime as normalizer -def _reference_source() -> str: - return ( - "import traceback\n" - "class Runtime:\n" - " def __init__(self):\n" - " self._values: dict[int, _Slot] = {}\n" - " def capture(self, error):\n" - " return ErrorInfo(\n" - ' "".join(traceback.format_exception(error)),\n' - " )\n" - " def store(self, handle, value):\n" - " self._values[handle] = value\n" - " def remove(self, handle):\n" - " del self._values[handle]\n" - " def lookup1(self, handle): return self._values.get(handle)\n" - " def lookup2(self, handle): return self._values.get(handle)\n" - " def lookup3(self, handle): return self._values.get(handle)\n" - " def lookup4(self, handle): return self._values.get(handle)\n" - " def lookup5(self, handle): return self._values.get(handle)\n" - " def lookup6(self, handle): return self._values.get(handle)\n" - " def lookup7(self, handle): return self._values.get(handle)\n" - " def lookup8(self, handle): return self._values.get(handle)\n" - " def callable(self, callable_handle):\n" - " return self._values.get(callable_handle)\n" - ) - - -def test_replaces_host_tracebacks_and_integer_handle_keys( +REFERENCE_SOURCE = Path("src/portapy/reference_api.py") + + +def test_normalizes_real_reference_runtime( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: path = tmp_path / "reference_api.py" - path.write_text(_reference_source(), encoding="utf-8") + path.write_text(REFERENCE_SOURCE.read_text(encoding="utf-8"), encoding="utf-8") monkeypatch.setattr(normalizer, "PATH", path) assert normalizer.main() == 0 @@ -47,10 +23,26 @@ def test_replaces_host_tracebacks_and_integer_handle_keys( assert "import traceback" not in result assert "format_exception" not in result assert 'type(error).__name__ + ": " + str(error)' in result + assert "dict[str, _Slot]" in result - assert result.count("self._values[str(handle)]") == 2 + assert "kind: ValueKind = ValueKind.INT" in result + assert "self._values[str(handle)] = _Slot(value, kind)" in result assert result.count("self._values.get(str(handle))") == 8 assert result.count("self._values.get(str(callable_handle))") == 1 + assert "del self._values[str(handle)]" in result + + assert "self._store(None, ValueKind.NONE)" in result + assert "self._store(value, ValueKind.BOOL)" in result + assert "self._store(value, ValueKind.INT)" in result + assert "self._store(value, ValueKind.FLOAT)" in result + assert "self._store(value, ValueKind.STRING)" in result + assert "self._store(value, ValueKind.BYTES)" in result + + assert "return Status.OK, slot.kind" in result + assert "slot.kind is not ValueKind.INT" in result + assert "slot.kind is not ValueKind.FLOAT" in result + assert "slot.kind is not ValueKind.STRING" in result + assert "type(slot.value)" not in result def test_rejects_unexpected_reference_runtime_shape( From c887525594c93721d13c95e063c5cc8be1724393 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:25:14 -0500 Subject: [PATCH 073/450] Normalize native traceback dictionary keys --- tools/normalize_full_core_tracebacks.py | 48 +++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tools/normalize_full_core_tracebacks.py diff --git a/tools/normalize_full_core_tracebacks.py b/tools/normalize_full_core_tracebacks.py new file mode 100644 index 00000000..f2a0f256 --- /dev/null +++ b/tools/normalize_full_core_tracebacks.py @@ -0,0 +1,48 @@ +"""Use native-safe string keys for synthetic traceback storage.""" +from __future__ import annotations + +from pathlib import Path + + +PATH = Path("src/portapy/core/vm.py") + +_REPLACEMENTS = ( + ( + ' self._synthetic_tracebacks: dict[int, "_PyTBProxy"] = {}', + ' self._synthetic_tracebacks: dict[str, "_PyTBProxy"] = {}', + "traceback table annotation", + ), + ( + "self._synthetic_tracebacks.get(id(target), target.__traceback__)", + "self._synthetic_tracebacks.get(str(id(target)), target.__traceback__)", + "traceback attribute lookup", + ), + ( + "prior = self._synthetic_tracebacks.get(id(exc))", + "prior = self._synthetic_tracebacks.get(str(id(exc)))", + "traceback prior lookup", + ), + ( + "self._synthetic_tracebacks[id(exc)] = _PyTBProxy(tb_frame, prior)", + "self._synthetic_tracebacks[str(id(exc))] = _PyTBProxy(tb_frame, prior)", + "traceback storage", + ), +) + + +def main() -> int: + source = PATH.read_text(encoding="utf-8") + for old, new, label in _REPLACEMENTS: + count = source.count(old) + if count != 1: + raise RuntimeError( + f"native {label}: expected one source form, found {count}" + ) + source = source.replace(old, new, 1) + PATH.write_text(source, encoding="utf-8") + print("NORMALIZED NATIVE TRACEBACK KEYS", len(_REPLACEMENTS)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 50ca0a66ffe7a97a3b61eb5b5662037b5ee6bb38 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:25:27 -0500 Subject: [PATCH 074/450] Test native traceback key normalization --- tests/test_normalize_full_core_tracebacks.py | 47 ++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tests/test_normalize_full_core_tracebacks.py diff --git a/tests/test_normalize_full_core_tracebacks.py b/tests/test_normalize_full_core_tracebacks.py new file mode 100644 index 00000000..16268e1a --- /dev/null +++ b/tests/test_normalize_full_core_tracebacks.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tools import normalize_full_core_tracebacks as normalizer + + +_SOURCE = '''class VirtualMachine: + def __init__(self) -> None: + self._synthetic_tracebacks: dict[int, "_PyTBProxy"] = {} + + def read(self, target): + return self._synthetic_tracebacks.get(id(target), target.__traceback__) + + def write(self, exc, tb_frame): + prior = self._synthetic_tracebacks.get(id(exc)) + self._synthetic_tracebacks[id(exc)] = _PyTBProxy(tb_frame, prior) +''' + + +def test_uses_string_keys_for_native_tracebacks( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "vm.py" + path.write_text(_SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + result = path.read_text(encoding="utf-8") + assert 'dict[str, "_PyTBProxy"]' in result + assert "get(str(id(target))" in result + assert "get(str(id(exc)))" in result + assert "[str(id(exc))]" in result + assert "[id(exc)]" not in result + + +def test_rejects_missing_traceback_storage_shape( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "vm.py" + path.write_text("class VirtualMachine: pass\n", encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + with pytest.raises(RuntimeError, match="traceback table annotation"): + normalizer.main() From 95367ad2bbbdfa4bac592dc1c9931f2a73c65472 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:25:51 -0500 Subject: [PATCH 075/450] Normalize native traceback keys in full build --- tools/normalize_full_core_validation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/normalize_full_core_validation.py b/tools/normalize_full_core_validation.py index 1b71a145..2fa188bd 100644 --- a/tools/normalize_full_core_validation.py +++ b/tools/normalize_full_core_validation.py @@ -23,6 +23,7 @@ from tools.normalize_full_core_opcode_maps import main as normalize_opcode_maps from tools.normalize_full_core_pattern_slices import main as normalize_pattern_slices from tools.normalize_full_core_probe import main as normalize_probe +from tools.normalize_full_core_tracebacks import main as normalize_tracebacks from tools.normalize_full_reference_abi_helpers import ( main as normalize_reference_abi_helpers, ) @@ -85,6 +86,7 @@ def main() -> int: normalize_extended_semantics() normalize_collections() normalize_builtins() + normalize_tracebacks() _normalize_nested_code_introspection() _normalize_opcode_validation() return 0 From 48a604244574ee70e5faa3441aa36fff88441b3c Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:29:05 -0500 Subject: [PATCH 076/450] Make Windows native toolchain setup resilient --- tools/install_windows_toolchain.ps1 | 81 +++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 tools/install_windows_toolchain.ps1 diff --git a/tools/install_windows_toolchain.ps1 b/tools/install_windows_toolchain.ps1 new file mode 100644 index 00000000..b44b7c47 --- /dev/null +++ b/tools/install_windows_toolchain.ps1 @@ -0,0 +1,81 @@ +$ErrorActionPreference = 'Stop' + +function Add-ToolDirectory([string] $ExecutablePath) { + $directory = Split-Path -Parent $ExecutablePath + if (-not ($env:PATH -split ';' | Where-Object { $_ -eq $directory })) { + $env:PATH = "$directory;$env:PATH" + if ($env:GITHUB_PATH) { + $directory | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + } + } +} + +function Find-Tool([string] $Name, [string[]] $Candidates) { + $command = Get-Command $Name -ErrorAction SilentlyContinue + if ($command) { + return $command.Source + } + foreach ($candidate in $Candidates) { + if (Test-Path $candidate) { + Add-ToolDirectory $candidate + return $candidate + } + } + return $null +} + +function Install-ChocolateyPackage([string] $Package) { + $successCodes = @(0, 1641, 3010) + for ($attempt = 1; $attempt -le 3; $attempt += 1) { + Write-Host "Installing $Package with Chocolatey (attempt $attempt/3)" + & choco install $Package -y --no-progress --limit-output + $status = $LASTEXITCODE + if ($successCodes -contains $status) { + return + } + if ($attempt -lt 3) { + Start-Sleep -Seconds (10 * $attempt) + } + } + throw "Chocolatey could not install $Package (exit code $status)" +} + +$nasmCandidates = @( + 'C:\Program Files\NASM\nasm.exe', + 'C:\ProgramData\chocolatey\bin\nasm.exe', + 'C:\tools\nasm\nasm.exe' +) +$gccCandidates = @( + 'C:\msys64\mingw64\bin\gcc.exe', + 'C:\mingw64\bin\gcc.exe', + 'C:\tools\mingw64\bin\gcc.exe', + 'C:\ProgramData\mingw64\mingw64\bin\gcc.exe' +) + +$nasm = Find-Tool 'nasm' $nasmCandidates +if (-not $nasm) { + Install-ChocolateyPackage 'nasm' + $nasm = Find-Tool 'nasm' $nasmCandidates +} +if (-not $nasm) { + throw 'NASM was not found after installation' +} + +$gcc = Find-Tool 'gcc' $gccCandidates +if (-not $gcc) { + Install-ChocolateyPackage 'mingw' + $gcc = Find-Tool 'gcc' $gccCandidates +} +if (-not $gcc) { + throw 'MinGW GCC was not found after installation' +} + +Add-ToolDirectory $nasm +Add-ToolDirectory $gcc + +Write-Host "NASM: $nasm" +& nasm -v +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +Write-Host "GCC: $gcc" +& gcc --version +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } From a7c0024d95af5ceebdd67f98dce4e84e55a1bd9e Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:29:33 -0500 Subject: [PATCH 077/450] Use resilient Windows toolchain setup --- .github/workflows/native-build-diagnostic.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/native-build-diagnostic.yml b/.github/workflows/native-build-diagnostic.yml index 0b512faa..70ebe51e 100644 --- a/.github/workflows/native-build-diagnostic.yml +++ b/.github/workflows/native-build-diagnostic.yml @@ -100,7 +100,7 @@ jobs: shell: powershell run: | $installLog = Join-Path $env:RUNNER_TEMP 'portapy-install-windows.log' - choco install nasm mingw -y --no-progress *> $installLog + & .\tools\install_windows_toolchain.ps1 *>> $installLog if ($LASTEXITCODE -ne 0) { Get-Content $installLog; exit $LASTEXITCODE } python -m pip install -e '.[test]' *>> $installLog if ($LASTEXITCODE -ne 0) { Get-Content $installLog; exit $LASTEXITCODE } @@ -145,6 +145,7 @@ jobs: with: name: native-build-diagnostic-windows path: | + ${{ runner.temp }}/portapy-install-windows.log ${{ runner.temp }}/portapy-build-windows.log ${{ runner.temp }}/portapy-report-windows.log ${{ runner.temp }}/portapy-build-windows From a21e06c0dba4b7043b22d8fe4675441bc3b6ff14 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:30:06 -0500 Subject: [PATCH 078/450] Stabilize Windows release toolchain setup --- .github/workflows/release-3.14.0.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-3.14.0.yml b/.github/workflows/release-3.14.0.yml index 1daf5d5c..be8c7c44 100644 --- a/.github/workflows/release-3.14.0.yml +++ b/.github/workflows/release-3.14.0.yml @@ -83,20 +83,24 @@ jobs: - name: Install package and native toolchain shell: powershell run: | - choco install nasm mingw -y --no-progress + & .\tools\install_windows_toolchain.ps1 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python -m pip install -e '.[test]' + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python -m pip install --no-cache-dir --force-reinstall ` 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Build full portapy.dll shell: powershell run: | New-Item -ItemType Directory -Force dist | Out-Null - python tools/build_native_typed.py ` + $output = & python tools/build_native_typed.py ` --target windows ` --output dist/portapy.dll ` --work-dir dist/build-windows ` - 2>&1 | Tee-Object -FilePath dist/build-windows.log + 2>&1 $buildStatus = $LASTEXITCODE + $output | Tee-Object -FilePath dist/build-windows.log if ($buildStatus -ne 0) { exit $buildStatus } - name: Validate Windows stable artifact shell: powershell From 2ff08978ba2d06108aa6b9086f6dac25825f7c9c Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:33:05 -0500 Subject: [PATCH 079/450] Route native type checks through tagged handles --- tools/normalize_full_reference_abi_helpers.py | 77 ++++++++++++++++++- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/tools/normalize_full_reference_abi_helpers.py b/tools/normalize_full_reference_abi_helpers.py index 2b2ed84a..b93eeeff 100644 --- a/tools/normalize_full_reference_abi_helpers.py +++ b/tools/normalize_full_reference_abi_helpers.py @@ -81,6 +81,36 @@ def __call__(self, name: str) -> object: return len(_runtimes) - 1 ''' +_VALUE_GET_KIND_SOURCE = ''' +instance = _runtime(runtime) +if instance is None: + _set_status(PORTAPY_INVALID_HANDLE) + return PORTAPY_VALUE_OBJECT +status, kind = instance.value_kind(value) +_set_status(status) +return int(kind) +''' + +_VALUE_AS_BOOL_SOURCE = ''' +instance = _runtime(runtime) +if instance is None: + _set_status(PORTAPY_INVALID_HANDLE) + return 0 +status, kind = instance.value_kind(value) +if status is not Status.OK: + _set_status(status) + return 0 +if kind is not ValueKind.BOOL: + _set_status(PORTAPY_TYPE_ERROR) + return 0 +status, target = instance.unbox(value) +if status is not Status.OK: + _set_status(status) + return 0 +_set_status(PORTAPY_OK) +return 1 if target else 0 +''' + def _is_utf8_source_upper_bound(node: ast.AST) -> bool: """Return True for ``source_size > len(source)``. @@ -114,6 +144,10 @@ def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST | None: self.generic_visit(node) if node.name == "_portapy_runtime_create_impl": node.body = ast.parse(_RUNTIME_CREATE_SOURCE).body + elif node.name == "_portapy_value_get_kind_impl": + node.body = ast.parse(_VALUE_GET_KIND_SOURCE).body + elif node.name == "_portapy_value_as_bool_impl": + node.body = ast.parse(_VALUE_AS_BOOL_SOURCE).body return node def visit_AsyncFunctionDef( @@ -184,18 +218,56 @@ def main() -> int: ), None, ) + value_get_kind = next( + ( + node + for node in verified.body + if isinstance(node, ast.FunctionDef) + and node.name == "_portapy_value_get_kind_impl" + ), + None, + ) + value_as_bool = next( + ( + node + for node in verified.body + if isinstance(node, ast.FunctionDef) + and node.name == "_portapy_value_as_bool_impl" + ), + None, + ) runtime_create_text = ast.unparse(runtime_create) if runtime_create is not None else "" + value_get_kind_text = ast.unparse(value_get_kind) if value_get_kind is not None else "" + value_as_bool_text = ast.unparse(value_as_bool) if value_as_bool is not None else "" loader_ready = ( "_PortaPyImportLoader" in classes and "__pyinbin_import__" in runtime_create_text ) builtins_ready = "_seed_builtins" in runtime_create_text - if missing or stale or unsafe_spans or not loader_ready or not builtins_ready: + tagged_kind_ready = ( + "instance.value_kind(value)" in value_get_kind_text + and "_value_kind(" not in value_get_kind_text + ) + tagged_bool_ready = ( + "instance.value_kind(value)" in value_as_bool_text + and "type(target)" not in value_as_bool_text + ) + if ( + missing + or stale + or unsafe_spans + or not loader_ready + or not builtins_ready + or not tagged_kind_ready + or not tagged_bool_ready + ): raise RuntimeError( "full Runtime ABI helper normalization failed; " f"missing={missing}, stale={stale}, " f"unsafe_utf8_spans={len(unsafe_spans)}, " - f"loader_ready={loader_ready}, builtins_ready={builtins_ready}" + f"loader_ready={loader_ready}, builtins_ready={builtins_ready}, " + f"tagged_kind_ready={tagged_kind_ready}, " + f"tagged_bool_ready={tagged_bool_ready}" ) print( "NORMALIZED FULL RUNTIME ABI HELPERS", @@ -203,6 +275,7 @@ def main() -> int: len(_DROP), "BUILTINS", "IMPORT_LOADER", + "TAGGED_VALUES", ) return 0 From be0223d6aac6c714ba24fdc63d8cd52debcefa62 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:33:29 -0500 Subject: [PATCH 080/450] Test tagged native ABI value checks --- .../test_full_reference_abi_normalization.py | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/test_full_reference_abi_normalization.py b/tests/test_full_reference_abi_normalization.py index 6c65a9b6..681dcc5e 100644 --- a/tests/test_full_reference_abi_normalization.py +++ b/tests/test_full_reference_abi_normalization.py @@ -7,6 +7,14 @@ from tools import normalize_full_reference_abi_helpers as normalizer +def _function(module: ast.Module, name: str) -> ast.FunctionDef: + return next( + node + for node in module.body + if isinstance(node, ast.FunctionDef) and node.name == name + ) + + def test_full_reference_normalization_installs_runtime_support( tmp_path: Path, monkeypatch, @@ -28,17 +36,23 @@ def test_full_reference_normalization_installs_runtime_support( assert "_PortaPyImportLoader" in classes assert "source_size > len(source)" not in source - runtime_create = next( - node - for node in module.body - if isinstance(node, ast.FunctionDef) - and node.name == "_portapy_runtime_create_impl" - ) - runtime_source = ast.unparse(runtime_create) + runtime_source = ast.unparse(_function(module, "_portapy_runtime_create_impl")) assert "instance._vm._seed_builtins(instance._globals)" in runtime_source assert "_PortaPyImportLoader(instance)" in runtime_source assert "__pyinbin_import__" in runtime_source + kind_source = ast.unparse(_function(module, "_portapy_value_get_kind_impl")) + assert "instance.value_kind(value)" in kind_source + assert "return int(kind)" in kind_source + assert "instance.unbox(value)" not in kind_source + assert "_value_kind(" not in kind_source + + bool_source = ast.unparse(_function(module, "_portapy_value_as_bool_impl")) + assert "instance.value_kind(value)" in bool_source + assert "kind is not ValueKind.BOOL" in bool_source + assert "instance.unbox(value)" in bool_source + assert "type(target)" not in bool_source + loader = next( node for node in module.body From 853ff55254cfa4ed3cb95b0bffb8de0ad02719f7 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:36:06 -0500 Subject: [PATCH 081/450] Tag ABI-created native values --- tools/normalize_full_reference_abi_helpers.py | 97 +++++++++++++------ 1 file changed, 66 insertions(+), 31 deletions(-) diff --git a/tools/normalize_full_reference_abi_helpers.py b/tools/normalize_full_reference_abi_helpers.py index b93eeeff..2ef5ccfb 100644 --- a/tools/normalize_full_reference_abi_helpers.py +++ b/tools/normalize_full_reference_abi_helpers.py @@ -50,6 +50,15 @@ "_portapy_cabi_list_append_impl": "_portapy_list_append_impl", } +_STORE_KIND_BY_FUNCTION = { + "_portapy_value_from_data_begin_impl": "kind", + "_portapy_value_from_host_object_impl": "ValueKind.OBJECT", + "_portapy_value_from_host_callable_impl": "ValueKind.CALLABLE", + "_portapy_tuple_begin_impl": "ValueKind.TUPLE", + "_portapy_dict_begin_impl": "ValueKind.DICT", + "_portapy_list_begin_impl": "ValueKind.LIST", +} + _IMPORT_LOADER_SOURCE = ''' class _PortaPyImportLoader: def __init__(self, instance: Runtime) -> None: @@ -136,6 +145,26 @@ def _is_utf8_source_upper_bound(node: ast.AST) -> bool: ) +class _StoreTagger(ast.NodeTransformer): + def __init__(self, kind_source: str) -> None: + self.kind = ast.parse(kind_source, mode="eval").body + self.count = 0 + + def visit_Call(self, node: ast.Call) -> ast.AST: + self.generic_visit(node) + if ( + isinstance(node.func, ast.Attribute) + and node.func.attr == "_store" + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "instance" + and len(node.args) == 1 + and not node.keywords + ): + node.args.append(ast.copy_location(self.kind, node.args[0])) + self.count += 1 + return node + + class _Rewrite(ast.NodeTransformer): def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST | None: if node.name in _DROP: @@ -148,6 +177,16 @@ def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST | None: node.body = ast.parse(_VALUE_GET_KIND_SOURCE).body elif node.name == "_portapy_value_as_bool_impl": node.body = ast.parse(_VALUE_AS_BOOL_SOURCE).body + + kind_source = _STORE_KIND_BY_FUNCTION.get(node.name) + if kind_source is not None: + tagger = _StoreTagger(kind_source) + node = tagger.visit(node) + if tagger.count != 1: + raise RuntimeError( + f"native store tagging for {node.name}: expected one call, " + f"found {tagger.count}" + ) return node def visit_AsyncFunctionDef( @@ -178,6 +217,18 @@ def visit_BoolOp(self, node: ast.BoolOp) -> ast.AST: return ast.copy_location(ast.BoolOp(op=node.op, values=values), node) +def _function_text(module: ast.Module, name: str) -> str: + function = next( + ( + node + for node in module.body + if isinstance(node, ast.FunctionDef) and node.name == name + ), + None, + ) + return ast.unparse(function) if function is not None else "" + + def main() -> int: module = ast.parse(PATH.read_text(encoding="utf-8")) module = _Rewrite().visit(module) @@ -209,36 +260,9 @@ def main() -> int: for node in ast.walk(verified) if _is_utf8_source_upper_bound(node) ] - runtime_create = next( - ( - node - for node in verified.body - if isinstance(node, ast.FunctionDef) - and node.name == "_portapy_runtime_create_impl" - ), - None, - ) - value_get_kind = next( - ( - node - for node in verified.body - if isinstance(node, ast.FunctionDef) - and node.name == "_portapy_value_get_kind_impl" - ), - None, - ) - value_as_bool = next( - ( - node - for node in verified.body - if isinstance(node, ast.FunctionDef) - and node.name == "_portapy_value_as_bool_impl" - ), - None, - ) - runtime_create_text = ast.unparse(runtime_create) if runtime_create is not None else "" - value_get_kind_text = ast.unparse(value_get_kind) if value_get_kind is not None else "" - value_as_bool_text = ast.unparse(value_as_bool) if value_as_bool is not None else "" + runtime_create_text = _function_text(verified, "_portapy_runtime_create_impl") + value_get_kind_text = _function_text(verified, "_portapy_value_get_kind_impl") + value_as_bool_text = _function_text(verified, "_portapy_value_as_bool_impl") loader_ready = ( "_PortaPyImportLoader" in classes and "__pyinbin_import__" in runtime_create_text @@ -252,6 +276,14 @@ def main() -> int: "instance.value_kind(value)" in value_as_bool_text and "type(target)" not in value_as_bool_text ) + tagged_stores_ready = all( + _function_text(verified, name).count("instance._store(") == 1 + and _function_text(verified, name).count(", ValueKind.") == 1 + for name in _STORE_KIND_BY_FUNCTION + if name != "_portapy_value_from_data_begin_impl" + ) and "instance._store(_DataBuilder(kind, size), kind)" in _function_text( + verified, "_portapy_value_from_data_begin_impl" + ) if ( missing or stale @@ -260,6 +292,7 @@ def main() -> int: or not builtins_ready or not tagged_kind_ready or not tagged_bool_ready + or not tagged_stores_ready ): raise RuntimeError( "full Runtime ABI helper normalization failed; " @@ -267,7 +300,8 @@ def main() -> int: f"unsafe_utf8_spans={len(unsafe_spans)}, " f"loader_ready={loader_ready}, builtins_ready={builtins_ready}, " f"tagged_kind_ready={tagged_kind_ready}, " - f"tagged_bool_ready={tagged_bool_ready}" + f"tagged_bool_ready={tagged_bool_ready}, " + f"tagged_stores_ready={tagged_stores_ready}" ) print( "NORMALIZED FULL RUNTIME ABI HELPERS", @@ -276,6 +310,7 @@ def main() -> int: "BUILTINS", "IMPORT_LOADER", "TAGGED_VALUES", + "TAGGED_STORES", ) return 0 From 14f83eef3474dec4cc79a356b22129bcac8a8586 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:36:32 -0500 Subject: [PATCH 082/450] Test tags for ABI-created values --- .../test_full_reference_abi_normalization.py | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/test_full_reference_abi_normalization.py b/tests/test_full_reference_abi_normalization.py index 681dcc5e..330b7e12 100644 --- a/tests/test_full_reference_abi_normalization.py +++ b/tests/test_full_reference_abi_normalization.py @@ -15,6 +15,10 @@ def _function(module: ast.Module, name: str) -> ast.FunctionDef: ) +def _source(module: ast.Module, name: str) -> str: + return ast.unparse(_function(module, name)) + + def test_full_reference_normalization_installs_runtime_support( tmp_path: Path, monkeypatch, @@ -36,23 +40,38 @@ def test_full_reference_normalization_installs_runtime_support( assert "_PortaPyImportLoader" in classes assert "source_size > len(source)" not in source - runtime_source = ast.unparse(_function(module, "_portapy_runtime_create_impl")) + runtime_source = _source(module, "_portapy_runtime_create_impl") assert "instance._vm._seed_builtins(instance._globals)" in runtime_source assert "_PortaPyImportLoader(instance)" in runtime_source assert "__pyinbin_import__" in runtime_source - kind_source = ast.unparse(_function(module, "_portapy_value_get_kind_impl")) + kind_source = _source(module, "_portapy_value_get_kind_impl") assert "instance.value_kind(value)" in kind_source assert "return int(kind)" in kind_source assert "instance.unbox(value)" not in kind_source assert "_value_kind(" not in kind_source - bool_source = ast.unparse(_function(module, "_portapy_value_as_bool_impl")) + bool_source = _source(module, "_portapy_value_as_bool_impl") assert "instance.value_kind(value)" in bool_source assert "kind is not ValueKind.BOOL" in bool_source assert "instance.unbox(value)" in bool_source assert "type(target)" not in bool_source + assert "instance._store(_DataBuilder(kind, size), kind)" in _source( + module, "_portapy_value_from_data_begin_impl" + ) + tagged_stores = { + "_portapy_value_from_host_object_impl": "ValueKind.OBJECT", + "_portapy_value_from_host_callable_impl": "ValueKind.CALLABLE", + "_portapy_tuple_begin_impl": "ValueKind.TUPLE", + "_portapy_dict_begin_impl": "ValueKind.DICT", + "_portapy_list_begin_impl": "ValueKind.LIST", + } + for function_name, kind in tagged_stores.items(): + function_source = _source(module, function_name) + assert "instance._store(" in function_source + assert kind in function_source + loader = next( node for node in module.body From 05adc8593b1cbb05eeb808895e3cfb3b074a1ca6 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:40:07 -0500 Subject: [PATCH 083/450] Run full reference normalization in probe test --- tests/test_full_core_probe.py | 49 +++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/tests/test_full_core_probe.py b/tests/test_full_core_probe.py index 625b13ad..d8e25548 100644 --- a/tests/test_full_core_probe.py +++ b/tests/test_full_core_probe.py @@ -1,19 +1,46 @@ +from __future__ import annotations + +import importlib +import sys + from tools.materialize_full_reference_entry import main as materialize_reference_entry from tools.normalize_full_reference_abi_helpers import ( main as normalize_reference_abi_helpers, ) +from tools.normalize_full_reference_runtime import ( + PATH as REFERENCE_RUNTIME_PATH, + main as normalize_reference_runtime, +) + + +_TEMPORARY_MODULES = ( + "portapy.native_full_core_probe", + "portapy.native_full_reference_entry", + "portapy.reference_api", +) def test_full_core_probe_executes_reference_abi_path() -> None: - materialize_reference_entry() - normalize_reference_abi_helpers() - from portapy.native_full_core_probe import portapy_full_core_probe - from portapy.native_full_reference_entry import _runtimes + original_reference_runtime = REFERENCE_RUNTIME_PATH.read_text(encoding="utf-8") + try: + normalize_reference_runtime() + materialize_reference_entry() + normalize_reference_abi_helpers() + from portapy.native_full_core_probe import portapy_full_core_probe + from portapy.native_full_reference_entry import _runtimes - result = portapy_full_core_probe() - errors = [ - runtime.last_error() - for runtime in _runtimes - if runtime is not None and runtime.last_error() is not None - ] - assert result == 42, errors + result = portapy_full_core_probe() + errors = [ + runtime.last_error() + for runtime in _runtimes + if runtime is not None and runtime.last_error() is not None + ] + assert result == 42, errors + finally: + REFERENCE_RUNTIME_PATH.write_text( + original_reference_runtime, + encoding="utf-8", + ) + for name in _TEMPORARY_MODULES: + sys.modules.pop(name, None) + importlib.invalidate_caches() From 1587f7f7905c59f1b5984fdb1deb31975bb3103a Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:41:30 -0500 Subject: [PATCH 084/450] Reload normalized reference runtime in probe test --- tests/test_full_core_probe.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_full_core_probe.py b/tests/test_full_core_probe.py index d8e25548..5b98f4c1 100644 --- a/tests/test_full_core_probe.py +++ b/tests/test_full_core_probe.py @@ -20,12 +20,20 @@ ) +def _unload_temporary_modules() -> None: + for name in _TEMPORARY_MODULES: + sys.modules.pop(name, None) + importlib.invalidate_caches() + + def test_full_core_probe_executes_reference_abi_path() -> None: original_reference_runtime = REFERENCE_RUNTIME_PATH.read_text(encoding="utf-8") try: + _unload_temporary_modules() normalize_reference_runtime() materialize_reference_entry() normalize_reference_abi_helpers() + _unload_temporary_modules() from portapy.native_full_core_probe import portapy_full_core_probe from portapy.native_full_reference_entry import _runtimes @@ -41,6 +49,4 @@ def test_full_core_probe_executes_reference_abi_path() -> None: original_reference_runtime, encoding="utf-8", ) - for name in _TEMPORARY_MODULES: - sys.modules.pop(name, None) - importlib.invalidate_caches() + _unload_temporary_modules() From e28889273b7c6a29025456e697eea88305a9f208 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:43:57 -0500 Subject: [PATCH 085/450] Retry pinned compiler installation on Windows --- .github/workflows/native-build-diagnostic.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/native-build-diagnostic.yml b/.github/workflows/native-build-diagnostic.yml index 70ebe51e..768d90bc 100644 --- a/.github/workflows/native-build-diagnostic.yml +++ b/.github/workflows/native-build-diagnostic.yml @@ -104,10 +104,18 @@ jobs: if ($LASTEXITCODE -ne 0) { Get-Content $installLog; exit $LASTEXITCODE } python -m pip install -e '.[test]' *>> $installLog if ($LASTEXITCODE -ne 0) { Get-Content $installLog; exit $LASTEXITCODE } - python -m pip install --no-cache-dir --force-reinstall ` - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' ` - *>> $installLog - if ($LASTEXITCODE -ne 0) { Get-Content $installLog; exit $LASTEXITCODE } + $compiler = 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' + $compilerInstalled = $false + for ($attempt = 1; $attempt -le 3; $attempt += 1) { + "Installing pinned asmpython (attempt $attempt/3)" | Out-File -FilePath $installLog -Encoding utf8 -Append + python -m pip install --no-cache-dir --force-reinstall $compiler *>> $installLog + if ($LASTEXITCODE -eq 0) { + $compilerInstalled = $true + break + } + if ($attempt -lt 3) { Start-Sleep -Seconds (10 * $attempt) } + } + if (-not $compilerInstalled) { Get-Content $installLog; exit 1 } - name: Capture compiler diagnostic id: build shell: powershell From ecadb2b91487bd85e14f6c9e19abbe927e4ecaa8 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:44:34 -0500 Subject: [PATCH 086/450] Retry pinned compiler installation in Windows release --- .github/workflows/release-3.14.0.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-3.14.0.yml b/.github/workflows/release-3.14.0.yml index be8c7c44..b08536b6 100644 --- a/.github/workflows/release-3.14.0.yml +++ b/.github/workflows/release-3.14.0.yml @@ -87,9 +87,18 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python -m pip install -e '.[test]' if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - python -m pip install --no-cache-dir --force-reinstall ` - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $compiler = 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' + $compilerInstalled = $false + for ($attempt = 1; $attempt -le 3; $attempt += 1) { + Write-Host "Installing pinned asmpython (attempt $attempt/3)" + python -m pip install --no-cache-dir --force-reinstall $compiler + if ($LASTEXITCODE -eq 0) { + $compilerInstalled = $true + break + } + if ($attempt -lt 3) { Start-Sleep -Seconds (10 * $attempt) } + } + if (-not $compilerInstalled) { exit 1 } - name: Build full portapy.dll shell: powershell run: | From 5f3a17028295537f2912bc80399ad2712a7e7cd6 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:46:58 -0500 Subject: [PATCH 087/450] Allow Windows compiler install retries --- .github/workflows/native-build-diagnostic.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/native-build-diagnostic.yml b/.github/workflows/native-build-diagnostic.yml index 768d90bc..c7ad4571 100644 --- a/.github/workflows/native-build-diagnostic.yml +++ b/.github/workflows/native-build-diagnostic.yml @@ -99,6 +99,7 @@ jobs: - name: Install quietly shell: powershell run: | + $PSNativeCommandUseErrorActionPreference = $false $installLog = Join-Path $env:RUNNER_TEMP 'portapy-install-windows.log' & .\tools\install_windows_toolchain.ps1 *>> $installLog if ($LASTEXITCODE -ne 0) { Get-Content $installLog; exit $LASTEXITCODE } From d2c22730806109d90cfae5bfb056e99b5a9681e5 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:47:35 -0500 Subject: [PATCH 088/450] Allow Windows release install retries --- .github/workflows/release-3.14.0.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-3.14.0.yml b/.github/workflows/release-3.14.0.yml index b08536b6..f966e51a 100644 --- a/.github/workflows/release-3.14.0.yml +++ b/.github/workflows/release-3.14.0.yml @@ -83,6 +83,7 @@ jobs: - name: Install package and native toolchain shell: powershell run: | + $PSNativeCommandUseErrorActionPreference = $false & .\tools\install_windows_toolchain.ps1 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python -m pip install -e '.[test]' From d7cb5abcfe28837759b91cc69a587e5a8a92638d Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:49:31 -0500 Subject: [PATCH 089/450] Install pinned asmpython with process-level retries --- tools/install_pinned_asmpython.ps1 | 50 ++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tools/install_pinned_asmpython.ps1 diff --git a/tools/install_pinned_asmpython.ps1 b/tools/install_pinned_asmpython.ps1 new file mode 100644 index 00000000..e19355e6 --- /dev/null +++ b/tools/install_pinned_asmpython.ps1 @@ -0,0 +1,50 @@ +param( + [string] $LogPath = '' +) + +$ErrorActionPreference = 'Stop' +$compiler = 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' +$python = (Get-Command python -ErrorAction Stop).Source + +function Write-InstallLog([string] $Text) { + Write-Host $Text + if ($LogPath) { + $Text | Out-File -FilePath $LogPath -Encoding utf8 -Append + } +} + +for ($attempt = 1; $attempt -le 3; $attempt += 1) { + Write-InstallLog "Installing pinned asmpython (attempt $attempt/3)" + $stdout = Join-Path $env:RUNNER_TEMP "asmpython-install-$attempt.stdout.log" + $stderr = Join-Path $env:RUNNER_TEMP "asmpython-install-$attempt.stderr.log" + Remove-Item $stdout, $stderr -Force -ErrorAction SilentlyContinue + + $process = Start-Process ` + -FilePath $python ` + -ArgumentList @( + '-m', 'pip', 'install', + '--no-cache-dir', '--force-reinstall', + $compiler + ) ` + -Wait ` + -PassThru ` + -NoNewWindow ` + -RedirectStandardOutput $stdout ` + -RedirectStandardError $stderr + + foreach ($path in @($stdout, $stderr)) { + if (Test-Path $path) { + Get-Content $path | ForEach-Object { Write-InstallLog $_ } + } + } + if ($process.ExitCode -eq 0) { + Write-InstallLog 'Pinned asmpython installation succeeded.' + exit 0 + } + Write-InstallLog "Pinned asmpython installation failed with exit code $($process.ExitCode)." + if ($attempt -lt 3) { + Start-Sleep -Seconds (10 * $attempt) + } +} + +throw 'Pinned asmpython installation failed after 3 attempts.' From cdb67a665213d61ba6e455244ef342c43786d1c2 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:50:08 -0500 Subject: [PATCH 090/450] Use process-based Windows compiler installer --- .github/workflows/native-build-diagnostic.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/native-build-diagnostic.yml b/.github/workflows/native-build-diagnostic.yml index c7ad4571..8a203d29 100644 --- a/.github/workflows/native-build-diagnostic.yml +++ b/.github/workflows/native-build-diagnostic.yml @@ -99,24 +99,12 @@ jobs: - name: Install quietly shell: powershell run: | - $PSNativeCommandUseErrorActionPreference = $false $installLog = Join-Path $env:RUNNER_TEMP 'portapy-install-windows.log' & .\tools\install_windows_toolchain.ps1 *>> $installLog if ($LASTEXITCODE -ne 0) { Get-Content $installLog; exit $LASTEXITCODE } python -m pip install -e '.[test]' *>> $installLog if ($LASTEXITCODE -ne 0) { Get-Content $installLog; exit $LASTEXITCODE } - $compiler = 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - $compilerInstalled = $false - for ($attempt = 1; $attempt -le 3; $attempt += 1) { - "Installing pinned asmpython (attempt $attempt/3)" | Out-File -FilePath $installLog -Encoding utf8 -Append - python -m pip install --no-cache-dir --force-reinstall $compiler *>> $installLog - if ($LASTEXITCODE -eq 0) { - $compilerInstalled = $true - break - } - if ($attempt -lt 3) { Start-Sleep -Seconds (10 * $attempt) } - } - if (-not $compilerInstalled) { Get-Content $installLog; exit 1 } + & .\tools\install_pinned_asmpython.ps1 -LogPath $installLog - name: Capture compiler diagnostic id: build shell: powershell From d3ecfb3b019426239bba141b18ec2af36d725f74 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:50:45 -0500 Subject: [PATCH 091/450] Use process-based Windows compiler installer in release --- .github/workflows/release-3.14.0.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/release-3.14.0.yml b/.github/workflows/release-3.14.0.yml index f966e51a..f3675282 100644 --- a/.github/workflows/release-3.14.0.yml +++ b/.github/workflows/release-3.14.0.yml @@ -83,23 +83,11 @@ jobs: - name: Install package and native toolchain shell: powershell run: | - $PSNativeCommandUseErrorActionPreference = $false & .\tools\install_windows_toolchain.ps1 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python -m pip install -e '.[test]' if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $compiler = 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - $compilerInstalled = $false - for ($attempt = 1; $attempt -le 3; $attempt += 1) { - Write-Host "Installing pinned asmpython (attempt $attempt/3)" - python -m pip install --no-cache-dir --force-reinstall $compiler - if ($LASTEXITCODE -eq 0) { - $compilerInstalled = $true - break - } - if ($attempt -lt 3) { Start-Sleep -Seconds (10 * $attempt) } - } - if (-not $compilerInstalled) { exit 1 } + & .\tools\install_pinned_asmpython.ps1 - name: Build full portapy.dll shell: powershell run: | From 3d87696016fdb0f806c0b2c90fbe89ce9ea7cc22 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:55:53 -0500 Subject: [PATCH 092/450] Normalize native enum values without __int__ --- tools/normalize_full_reference_abi_helpers.py | 58 ++++++++++++++++--- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/tools/normalize_full_reference_abi_helpers.py b/tools/normalize_full_reference_abi_helpers.py index 2ef5ccfb..60f7e12b 100644 --- a/tools/normalize_full_reference_abi_helpers.py +++ b/tools/normalize_full_reference_abi_helpers.py @@ -86,37 +86,37 @@ def __call__(self, name: str) -> object: instance._vm._seed_builtins(instance._globals) instance.set_global("__pyinbin_import__", _PortaPyImportLoader(instance)) _runtimes.append(instance) -_set_status(PORTAPY_OK) +_set_status(Status.OK) return len(_runtimes) - 1 ''' _VALUE_GET_KIND_SOURCE = ''' instance = _runtime(runtime) if instance is None: - _set_status(PORTAPY_INVALID_HANDLE) - return PORTAPY_VALUE_OBJECT + _set_status(Status.INVALID_HANDLE) + return ValueKind.OBJECT.value status, kind = instance.value_kind(value) _set_status(status) -return int(kind) +return kind.value ''' _VALUE_AS_BOOL_SOURCE = ''' instance = _runtime(runtime) if instance is None: - _set_status(PORTAPY_INVALID_HANDLE) + _set_status(Status.INVALID_HANDLE) return 0 status, kind = instance.value_kind(value) if status is not Status.OK: _set_status(status) return 0 if kind is not ValueKind.BOOL: - _set_status(PORTAPY_TYPE_ERROR) + _set_status(Status.TYPE_ERROR) return 0 status, target = instance.unbox(value) if status is not Status.OK: _set_status(status) return 0 -_set_status(PORTAPY_OK) +_set_status(Status.OK) return 1 if target else 0 ''' @@ -145,6 +145,16 @@ def _is_utf8_source_upper_bound(node: ast.AST) -> bool: ) +def _is_native_enum_expression(node: ast.AST) -> bool: + if isinstance(node, ast.Name): + return node.id in {"status", "kind"} + if not isinstance(node, ast.Attribute): + return False + if node.attr == "status": + return True + return isinstance(node.value, ast.Name) and node.value.id in {"Status", "ValueKind"} + + class _StoreTagger(ast.NodeTransformer): def __init__(self, kind_source: str) -> None: self.kind = ast.parse(kind_source, mode="eval").body @@ -205,6 +215,21 @@ def visit_Name(self, node: ast.Name) -> ast.AST: return node return ast.copy_location(ast.Name(id=renamed, ctx=node.ctx), node) + def visit_Call(self, node: ast.Call) -> ast.AST: + self.generic_visit(node) + if ( + isinstance(node.func, ast.Name) + and node.func.id == "int" + and len(node.args) == 1 + and not node.keywords + and _is_native_enum_expression(node.args[0]) + ): + return ast.copy_location( + ast.Attribute(value=node.args[0], attr="value", ctx=ast.Load()), + node, + ) + return node + def visit_BoolOp(self, node: ast.BoolOp) -> ast.AST: self.generic_visit(node) if not isinstance(node.op, ast.Or): @@ -260,7 +285,17 @@ def main() -> int: for node in ast.walk(verified) if _is_utf8_source_upper_bound(node) ] + enum_int_calls = [ + node + for node in ast.walk(verified) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "int" + and len(node.args) == 1 + and _is_native_enum_expression(node.args[0]) + ] runtime_create_text = _function_text(verified, "_portapy_runtime_create_impl") + set_status_text = _function_text(verified, "_set_status") value_get_kind_text = _function_text(verified, "_portapy_value_get_kind_impl") value_as_bool_text = _function_text(verified, "_portapy_value_as_bool_impl") loader_ready = ( @@ -268,6 +303,11 @@ def main() -> int: and "__pyinbin_import__" in runtime_create_text ) builtins_ready = "_seed_builtins" in runtime_create_text + enum_values_ready = ( + not enum_int_calls + and "status.value" in set_status_text + and "return kind.value" in value_get_kind_text + ) tagged_kind_ready = ( "instance.value_kind(value)" in value_get_kind_text and "_value_kind(" not in value_get_kind_text @@ -290,6 +330,7 @@ def main() -> int: or unsafe_spans or not loader_ready or not builtins_ready + or not enum_values_ready or not tagged_kind_ready or not tagged_bool_ready or not tagged_stores_ready @@ -298,7 +339,9 @@ def main() -> int: "full Runtime ABI helper normalization failed; " f"missing={missing}, stale={stale}, " f"unsafe_utf8_spans={len(unsafe_spans)}, " + f"enum_int_calls={len(enum_int_calls)}, " f"loader_ready={loader_ready}, builtins_ready={builtins_ready}, " + f"enum_values_ready={enum_values_ready}, " f"tagged_kind_ready={tagged_kind_ready}, " f"tagged_bool_ready={tagged_bool_ready}, " f"tagged_stores_ready={tagged_stores_ready}" @@ -309,6 +352,7 @@ def main() -> int: len(_DROP), "BUILTINS", "IMPORT_LOADER", + "ENUM_VALUES", "TAGGED_VALUES", "TAGGED_STORES", ) From 990347a5272c3eaf5e0bbc7dc86b235586aa5d3f Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:56:19 -0500 Subject: [PATCH 093/450] Test direct native enum value access --- tests/test_full_reference_abi_normalization.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_full_reference_abi_normalization.py b/tests/test_full_reference_abi_normalization.py index 330b7e12..59cac1a6 100644 --- a/tests/test_full_reference_abi_normalization.py +++ b/tests/test_full_reference_abi_normalization.py @@ -44,10 +44,16 @@ def test_full_reference_normalization_installs_runtime_support( assert "instance._vm._seed_builtins(instance._globals)" in runtime_source assert "_PortaPyImportLoader(instance)" in runtime_source assert "__pyinbin_import__" in runtime_source + assert "_set_status(Status.OK)" in runtime_source + + set_status_source = _source(module, "_set_status") + assert "status.value" in set_status_source + assert "int(status)" not in set_status_source kind_source = _source(module, "_portapy_value_get_kind_impl") assert "instance.value_kind(value)" in kind_source - assert "return int(kind)" in kind_source + assert "return kind.value" in kind_source + assert "int(kind)" not in kind_source assert "instance.unbox(value)" not in kind_source assert "_value_kind(" not in kind_source @@ -56,6 +62,7 @@ def test_full_reference_normalization_installs_runtime_support( assert "kind is not ValueKind.BOOL" in bool_source assert "instance.unbox(value)" in bool_source assert "type(target)" not in bool_source + assert "PORTAPY_" not in bool_source assert "instance._store(_DataBuilder(kind, size), kind)" in _source( module, "_portapy_value_from_data_begin_impl" From 502b0303ea55cde0c0d1723d427cb8d14fd95cba Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:00:25 -0500 Subject: [PATCH 094/450] Normalize raw ABI status constants --- tools/normalize_full_reference_abi_helpers.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tools/normalize_full_reference_abi_helpers.py b/tools/normalize_full_reference_abi_helpers.py index 60f7e12b..e7730722 100644 --- a/tools/normalize_full_reference_abi_helpers.py +++ b/tools/normalize_full_reference_abi_helpers.py @@ -59,6 +59,18 @@ "_portapy_list_begin_impl": "ValueKind.LIST", } +_STATUS_CONSTANTS = { + "PORTAPY_OK": "OK", + "PORTAPY_INVALID_ARGUMENT": "INVALID_ARGUMENT", + "PORTAPY_COMPILE_ERROR": "COMPILE_ERROR", + "PORTAPY_RUNTIME_ERROR": "RUNTIME_ERROR", + "PORTAPY_TYPE_ERROR": "TYPE_ERROR", + "PORTAPY_NOT_FOUND": "NOT_FOUND", + "PORTAPY_CLOSED": "CLOSED", + "PORTAPY_INVALID_HANDLE": "INVALID_HANDLE", + "PORTAPY_INTERRUPTED": "INTERRUPTED", +} + _IMPORT_LOADER_SOURCE = ''' class _PortaPyImportLoader: def __init__(self, instance: Runtime) -> None: @@ -217,6 +229,23 @@ def visit_Name(self, node: ast.Name) -> ast.AST: def visit_Call(self, node: ast.Call) -> ast.AST: self.generic_visit(node) + if ( + isinstance(node.func, ast.Name) + and node.func.id == "_set_status" + and len(node.args) == 1 + and not node.keywords + and isinstance(node.args[0], ast.Name) + and node.args[0].id in _STATUS_CONSTANTS + ): + node.args[0] = ast.copy_location( + ast.Attribute( + value=ast.Name(id="Status", ctx=ast.Load()), + attr=_STATUS_CONSTANTS[node.args[0].id], + ctx=ast.Load(), + ), + node.args[0], + ) + return node if ( isinstance(node.func, ast.Name) and node.func.id == "int" @@ -294,6 +323,16 @@ def main() -> int: and len(node.args) == 1 and _is_native_enum_expression(node.args[0]) ] + raw_status_calls = [ + node + for node in ast.walk(verified) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "_set_status" + and len(node.args) == 1 + and isinstance(node.args[0], ast.Name) + and node.args[0].id in _STATUS_CONSTANTS + ] runtime_create_text = _function_text(verified, "_portapy_runtime_create_impl") set_status_text = _function_text(verified, "_set_status") value_get_kind_text = _function_text(verified, "_portapy_value_get_kind_impl") @@ -305,6 +344,7 @@ def main() -> int: builtins_ready = "_seed_builtins" in runtime_create_text enum_values_ready = ( not enum_int_calls + and not raw_status_calls and "status.value" in set_status_text and "return kind.value" in value_get_kind_text ) @@ -340,6 +380,7 @@ def main() -> int: f"missing={missing}, stale={stale}, " f"unsafe_utf8_spans={len(unsafe_spans)}, " f"enum_int_calls={len(enum_int_calls)}, " + f"raw_status_calls={len(raw_status_calls)}, " f"loader_ready={loader_ready}, builtins_ready={builtins_ready}, " f"enum_values_ready={enum_values_ready}, " f"tagged_kind_ready={tagged_kind_ready}, " @@ -353,6 +394,7 @@ def main() -> int: "BUILTINS", "IMPORT_LOADER", "ENUM_VALUES", + "STATUS_VALUES", "TAGGED_VALUES", "TAGGED_STORES", ) From 3327db05451d30217c1b398b32e81a341b81845d Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:00:54 -0500 Subject: [PATCH 095/450] Test normalized raw status constants --- tests/test_full_reference_abi_normalization.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_full_reference_abi_normalization.py b/tests/test_full_reference_abi_normalization.py index 59cac1a6..65da5600 100644 --- a/tests/test_full_reference_abi_normalization.py +++ b/tests/test_full_reference_abi_normalization.py @@ -39,6 +39,7 @@ def test_full_reference_normalization_installs_runtime_support( } assert "_PortaPyImportLoader" in classes assert "source_size > len(source)" not in source + assert "_set_status(PORTAPY_" not in source runtime_source = _source(module, "_portapy_runtime_create_impl") assert "instance._vm._seed_builtins(instance._globals)" in runtime_source @@ -64,6 +65,10 @@ def test_full_reference_normalization_installs_runtime_support( assert "type(target)" not in bool_source assert "PORTAPY_" not in bool_source + list_source = _source(module, "_portapy_list_begin_impl") + assert "_set_status(Status.OK)" in list_source + assert "_set_status(PORTAPY_OK)" not in list_source + assert "instance._store(_DataBuilder(kind, size), kind)" in _source( module, "_portapy_value_from_data_begin_impl" ) From 4b6e1cdd3e166006ee9adc11322803c0546cf3db Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:03:01 -0500 Subject: [PATCH 096/450] Normalize conditional ABI status expressions --- tools/normalize_full_reference_abi_helpers.py | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/tools/normalize_full_reference_abi_helpers.py b/tools/normalize_full_reference_abi_helpers.py index e7730722..db9ba6ca 100644 --- a/tools/normalize_full_reference_abi_helpers.py +++ b/tools/normalize_full_reference_abi_helpers.py @@ -167,6 +167,28 @@ def _is_native_enum_expression(node: ast.AST) -> bool: return isinstance(node.value, ast.Name) and node.value.id in {"Status", "ValueKind"} +def _contains_raw_status(node: ast.AST) -> bool: + return any( + isinstance(item, ast.Name) and item.id in _STATUS_CONSTANTS + for item in ast.walk(node) + ) + + +class _StatusConstantRewriter(ast.NodeTransformer): + def visit_Name(self, node: ast.Name) -> ast.AST: + member = _STATUS_CONSTANTS.get(node.id) + if member is None: + return node + return ast.copy_location( + ast.Attribute( + value=ast.Name(id="Status", ctx=ast.Load()), + attr=member, + ctx=ast.Load(), + ), + node, + ) + + class _StoreTagger(ast.NodeTransformer): def __init__(self, kind_source: str) -> None: self.kind = ast.parse(kind_source, mode="eval").body @@ -234,17 +256,8 @@ def visit_Call(self, node: ast.Call) -> ast.AST: and node.func.id == "_set_status" and len(node.args) == 1 and not node.keywords - and isinstance(node.args[0], ast.Name) - and node.args[0].id in _STATUS_CONSTANTS ): - node.args[0] = ast.copy_location( - ast.Attribute( - value=ast.Name(id="Status", ctx=ast.Load()), - attr=_STATUS_CONSTANTS[node.args[0].id], - ctx=ast.Load(), - ), - node.args[0], - ) + node.args[0] = _StatusConstantRewriter().visit(node.args[0]) return node if ( isinstance(node.func, ast.Name) @@ -330,8 +343,7 @@ def main() -> int: and isinstance(node.func, ast.Name) and node.func.id == "_set_status" and len(node.args) == 1 - and isinstance(node.args[0], ast.Name) - and node.args[0].id in _STATUS_CONSTANTS + and _contains_raw_status(node.args[0]) ] runtime_create_text = _function_text(verified, "_portapy_runtime_create_impl") set_status_text = _function_text(verified, "_set_status") From dbbafb056b5f9da65c6dc37f4bde3fc6efcc029b Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:08:58 -0500 Subject: [PATCH 097/450] Map native enums without attribute access --- tools/normalize_full_reference_abi_helpers.py | 171 ++++++++++++++---- 1 file changed, 138 insertions(+), 33 deletions(-) diff --git a/tools/normalize_full_reference_abi_helpers.py b/tools/normalize_full_reference_abi_helpers.py index db9ba6ca..4e498295 100644 --- a/tools/normalize_full_reference_abi_helpers.py +++ b/tools/normalize_full_reference_abi_helpers.py @@ -71,6 +71,64 @@ "PORTAPY_INTERRUPTED": "INTERRUPTED", } +_NATIVE_ENUM_HELPERS_SOURCE = ''' +def _native_status_code(status: object) -> int: + if status is Status.OK: + return PORTAPY_OK + if status is Status.INVALID_ARGUMENT: + return PORTAPY_INVALID_ARGUMENT + if status is Status.COMPILE_ERROR: + return PORTAPY_COMPILE_ERROR + if status is Status.RUNTIME_ERROR: + return PORTAPY_RUNTIME_ERROR + if status is Status.TYPE_ERROR: + return PORTAPY_TYPE_ERROR + if status is Status.NOT_FOUND: + return PORTAPY_NOT_FOUND + if status is Status.CLOSED: + return PORTAPY_CLOSED + if status is Status.INVALID_HANDLE: + return PORTAPY_INVALID_HANDLE + if status is Status.INTERRUPTED: + return PORTAPY_INTERRUPTED + return PORTAPY_RUNTIME_ERROR + + +def _native_value_kind_code(kind: object) -> int: + if kind is ValueKind.NONE: + return PORTAPY_VALUE_NONE + if kind is ValueKind.BOOL: + return PORTAPY_VALUE_BOOL + if kind is ValueKind.INT: + return PORTAPY_VALUE_INT + if kind is ValueKind.FLOAT: + return PORTAPY_VALUE_FLOAT + if kind is ValueKind.STRING: + return PORTAPY_VALUE_STRING + if kind is ValueKind.BYTES: + return PORTAPY_VALUE_BYTES + if kind is ValueKind.CALLABLE: + return PORTAPY_VALUE_CALLABLE + if kind is ValueKind.TUPLE: + return PORTAPY_VALUE_TUPLE + if kind is ValueKind.DICT: + return PORTAPY_VALUE_DICT + if kind is ValueKind.LIST: + return PORTAPY_VALUE_LIST + return PORTAPY_VALUE_OBJECT + + +def _set_status_code(value: int) -> int: + _last_status[0] = value + return value +''' + +_SET_STATUS_SOURCE = ''' +value = _native_status_code(status) +_last_status[0] = value +return value +''' + _IMPORT_LOADER_SOURCE = ''' class _PortaPyImportLoader: def __init__(self, instance: Runtime) -> None: @@ -106,10 +164,10 @@ def __call__(self, name: str) -> object: instance = _runtime(runtime) if instance is None: _set_status(Status.INVALID_HANDLE) - return ValueKind.OBJECT.value + return PORTAPY_VALUE_OBJECT status, kind = instance.value_kind(value) _set_status(status) -return kind.value +return _native_value_kind_code(kind) ''' _VALUE_AS_BOOL_SOURCE = ''' @@ -157,14 +215,23 @@ def _is_utf8_source_upper_bound(node: ast.AST) -> bool: ) -def _is_native_enum_expression(node: ast.AST) -> bool: +def _native_enum_helper(node: ast.AST) -> str | None: if isinstance(node, ast.Name): - return node.id in {"status", "kind"} + if node.id == "kind": + return "_native_value_kind_code" + if node.id == "status": + return "_native_status_code" + return None if not isinstance(node, ast.Attribute): - return False + return None if node.attr == "status": - return True - return isinstance(node.value, ast.Name) and node.value.id in {"Status", "ValueKind"} + return "_native_status_code" + if isinstance(node.value, ast.Name): + if node.value.id == "Status": + return "_native_status_code" + if node.value.id == "ValueKind": + return "_native_value_kind_code" + return None def _contains_raw_status(node: ast.AST) -> bool: @@ -174,19 +241,28 @@ def _contains_raw_status(node: ast.AST) -> bool: ) -class _StatusConstantRewriter(ast.NodeTransformer): - def visit_Name(self, node: ast.Name) -> ast.AST: - member = _STATUS_CONSTANTS.get(node.id) - if member is None: - return node - return ast.copy_location( - ast.Attribute( - value=ast.Name(id="Status", ctx=ast.Load()), - attr=member, - ctx=ast.Load(), - ), - node, - ) +def _contains_native_enum_value_access(node: ast.AST) -> bool: + return any( + isinstance(item, ast.Attribute) + and item.attr == "value" + and _native_enum_helper(item.value) is not None + for item in ast.walk(node) + ) + + +class _RawDispatchStatusRewriter(ast.NodeTransformer): + def visit_Call(self, node: ast.Call) -> ast.AST: + self.generic_visit(node) + if ( + isinstance(node.func, ast.Name) + and node.func.id == "_set_status" + and len(node.args) == 1 + and not node.keywords + and isinstance(node.args[0], ast.Name) + and node.args[0].id == "status" + ): + node.func.id = "_set_status_code" + return node class _StoreTagger(ast.NodeTransformer): @@ -215,12 +291,16 @@ def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST | None: return None node.name = _RENAME.get(node.name, node.name) self.generic_visit(node) - if node.name == "_portapy_runtime_create_impl": + if node.name == "_set_status": + node.body = ast.parse(_SET_STATUS_SOURCE).body + elif node.name == "_portapy_runtime_create_impl": node.body = ast.parse(_RUNTIME_CREATE_SOURCE).body elif node.name == "_portapy_value_get_kind_impl": node.body = ast.parse(_VALUE_GET_KIND_SOURCE).body elif node.name == "_portapy_value_as_bool_impl": node.body = ast.parse(_VALUE_AS_BOOL_SOURCE).body + elif node.name == "_portapy_host_dispatch_complete_impl": + node = _RawDispatchStatusRewriter().visit(node) kind_source = _STORE_KIND_BY_FUNCTION.get(node.name) if kind_source is not None: @@ -256,20 +336,26 @@ def visit_Call(self, node: ast.Call) -> ast.AST: and node.func.id == "_set_status" and len(node.args) == 1 and not node.keywords + and _contains_raw_status(node.args[0]) ): - node.args[0] = _StatusConstantRewriter().visit(node.args[0]) + node.func.id = "_set_status_code" return node if ( isinstance(node.func, ast.Name) and node.func.id == "int" and len(node.args) == 1 and not node.keywords - and _is_native_enum_expression(node.args[0]) ): - return ast.copy_location( - ast.Attribute(value=node.args[0], attr="value", ctx=ast.Load()), - node, - ) + helper = _native_enum_helper(node.args[0]) + if helper is not None: + return ast.copy_location( + ast.Call( + func=ast.Name(id=helper, ctx=ast.Load()), + args=node.args, + keywords=[], + ), + node, + ) return node def visit_BoolOp(self, node: ast.BoolOp) -> ast.AST: @@ -304,6 +390,7 @@ def main() -> int: for node in module.body ): raise RuntimeError("full Runtime already contains the PortaPy import loader") + module.body.extend(ast.parse(_NATIVE_ENUM_HELPERS_SOURCE).body) module.body.extend(ast.parse(_IMPORT_LOADER_SOURCE).body) ast.fix_missing_locations(module) source = ast.unparse(module) + "\n" @@ -334,7 +421,7 @@ def main() -> int: and isinstance(node.func, ast.Name) and node.func.id == "int" and len(node.args) == 1 - and _is_native_enum_expression(node.args[0]) + and _native_enum_helper(node.args[0]) is not None ] raw_status_calls = [ node @@ -345,10 +432,18 @@ def main() -> int: and len(node.args) == 1 and _contains_raw_status(node.args[0]) ] + enum_value_accesses = [ + node + for node in ast.walk(verified) + if _contains_native_enum_value_access(node) + ] runtime_create_text = _function_text(verified, "_portapy_runtime_create_impl") set_status_text = _function_text(verified, "_set_status") + status_code_text = _function_text(verified, "_native_status_code") + kind_code_text = _function_text(verified, "_native_value_kind_code") value_get_kind_text = _function_text(verified, "_portapy_value_get_kind_impl") value_as_bool_text = _function_text(verified, "_portapy_value_as_bool_impl") + host_dispatch_text = _function_text(verified, "_portapy_host_dispatch_complete_impl") loader_ready = ( "_PortaPyImportLoader" in classes and "__pyinbin_import__" in runtime_create_text @@ -356,9 +451,16 @@ def main() -> int: builtins_ready = "_seed_builtins" in runtime_create_text enum_values_ready = ( not enum_int_calls - and not raw_status_calls - and "status.value" in set_status_text - and "return kind.value" in value_get_kind_text + and not enum_value_accesses + and "_native_status_code(status)" in set_status_text + and "_native_value_kind_code(kind)" in value_get_kind_text + and "PORTAPY_OK" in status_code_text + and "PORTAPY_VALUE_NONE" in kind_code_text + ) + status_paths_ready = ( + not raw_status_calls + and "_set_status_code(status)" in host_dispatch_text + and "_set_status(status)" not in host_dispatch_text ) tagged_kind_ready = ( "instance.value_kind(value)" in value_get_kind_text @@ -383,6 +485,7 @@ def main() -> int: or not loader_ready or not builtins_ready or not enum_values_ready + or not status_paths_ready or not tagged_kind_ready or not tagged_bool_ready or not tagged_stores_ready @@ -392,9 +495,11 @@ def main() -> int: f"missing={missing}, stale={stale}, " f"unsafe_utf8_spans={len(unsafe_spans)}, " f"enum_int_calls={len(enum_int_calls)}, " + f"enum_value_accesses={len(enum_value_accesses)}, " f"raw_status_calls={len(raw_status_calls)}, " f"loader_ready={loader_ready}, builtins_ready={builtins_ready}, " f"enum_values_ready={enum_values_ready}, " + f"status_paths_ready={status_paths_ready}, " f"tagged_kind_ready={tagged_kind_ready}, " f"tagged_bool_ready={tagged_bool_ready}, " f"tagged_stores_ready={tagged_stores_ready}" @@ -405,8 +510,8 @@ def main() -> int: len(_DROP), "BUILTINS", "IMPORT_LOADER", - "ENUM_VALUES", - "STATUS_VALUES", + "ENUM_CODES", + "STATUS_CODES", "TAGGED_VALUES", "TAGGED_STORES", ) From b46853e150ed78d55b4e89c140f4f9704188fb51 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:09:36 -0500 Subject: [PATCH 098/450] Test native enum code mapping --- .../test_full_reference_abi_normalization.py | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/test_full_reference_abi_normalization.py b/tests/test_full_reference_abi_normalization.py index 65da5600..1203b471 100644 --- a/tests/test_full_reference_abi_normalization.py +++ b/tests/test_full_reference_abi_normalization.py @@ -40,6 +40,8 @@ def test_full_reference_normalization_installs_runtime_support( assert "_PortaPyImportLoader" in classes assert "source_size > len(source)" not in source assert "_set_status(PORTAPY_" not in source + assert "status.value" not in source + assert "kind.value" not in source runtime_source = _source(module, "_portapy_runtime_create_impl") assert "instance._vm._seed_builtins(instance._globals)" in runtime_source @@ -48,12 +50,22 @@ def test_full_reference_normalization_installs_runtime_support( assert "_set_status(Status.OK)" in runtime_source set_status_source = _source(module, "_set_status") - assert "status.value" in set_status_source + assert "_native_status_code(status)" in set_status_source assert "int(status)" not in set_status_source + status_code_source = _source(module, "_native_status_code") + assert "status is Status.OK" in status_code_source + assert "return PORTAPY_OK" in status_code_source + assert "status.value" not in status_code_source + + value_kind_code_source = _source(module, "_native_value_kind_code") + assert "kind is ValueKind.NONE" in value_kind_code_source + assert "return PORTAPY_VALUE_NONE" in value_kind_code_source + assert "kind.value" not in value_kind_code_source + kind_source = _source(module, "_portapy_value_get_kind_impl") assert "instance.value_kind(value)" in kind_source - assert "return kind.value" in kind_source + assert "return _native_value_kind_code(kind)" in kind_source assert "int(kind)" not in kind_source assert "instance.unbox(value)" not in kind_source assert "_value_kind(" not in kind_source @@ -66,9 +78,13 @@ def test_full_reference_normalization_installs_runtime_support( assert "PORTAPY_" not in bool_source list_source = _source(module, "_portapy_list_begin_impl") - assert "_set_status(Status.OK)" in list_source + assert "_set_status_code(PORTAPY_OK)" in list_source assert "_set_status(PORTAPY_OK)" not in list_source + dispatch_source = _source(module, "_portapy_host_dispatch_complete_impl") + assert "_set_status_code(status)" in dispatch_source + assert "_set_status(status)" not in dispatch_source + assert "instance._store(_DataBuilder(kind, size), kind)" in _source( module, "_portapy_value_from_data_begin_impl" ) From f84af700012bdf04dab00f2d82211e089bead7d0 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:14:38 -0500 Subject: [PATCH 099/450] Avoid native exception introspection in core capture --- tools/normalize_full_reference_runtime.py | 24 ++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/tools/normalize_full_reference_runtime.py b/tools/normalize_full_reference_runtime.py index 349de7bc..ecf6b5bb 100644 --- a/tools/normalize_full_reference_runtime.py +++ b/tools/normalize_full_reference_runtime.py @@ -7,10 +7,24 @@ PATH = Path("src/portapy/reference_api.py") _TRACEBACK_IMPORT = "import traceback\n" -_TRACEBACK_FORMAT = ' "".join(traceback.format_exception(error)),\n' -_NATIVE_FORMAT = ( - ' type(error).__name__ + ": " + str(error),\n' -) +_CAPTURE_METHOD = ''' def _capture(self, status: Status, error: BaseException) -> Status: + self._last_error = ErrorInfo( + status, + type(error).__name__, + str(error), + "".join(traceback.format_exception(error)), + ) + return status +''' +_NATIVE_CAPTURE_METHOD = ''' def _capture(self, status: Status, error: BaseException) -> Status: + self._last_error = ErrorInfo( + status, + "PortaPyError", + "PortaPy operation failed", + "PortaPy operation failed", + ) + return status +''' _SLOT = '''@dataclass class _Slot: value: object @@ -188,7 +202,7 @@ def main() -> int: source = PATH.read_text(encoding="utf-8") replacements = ( (_TRACEBACK_IMPORT, "", "traceback import"), - (_TRACEBACK_FORMAT, _NATIVE_FORMAT, "traceback formatter"), + (_CAPTURE_METHOD, _NATIVE_CAPTURE_METHOD, "error capture"), (_SLOT, _NATIVE_SLOT, "value slot"), (_STORE, _NATIVE_STORE, "value store"), (_BOX_METHODS, _NATIVE_BOX_METHODS, "scalar boxing"), From 4390fdf196a77635f9da47c070be1ee3c76d0451 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:15:44 -0500 Subject: [PATCH 100/450] Test safe native error capture --- tests/test_normalize_full_reference_runtime.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_normalize_full_reference_runtime.py b/tests/test_normalize_full_reference_runtime.py index 1ed47f84..c5a673b7 100644 --- a/tests/test_normalize_full_reference_runtime.py +++ b/tests/test_normalize_full_reference_runtime.py @@ -22,7 +22,10 @@ def test_normalizes_real_reference_runtime( result = path.read_text(encoding="utf-8") assert "import traceback" not in result assert "format_exception" not in result - assert 'type(error).__name__ + ": " + str(error)' in result + assert "type(error).__name__" not in result + assert "str(error)" not in result + assert '"PortaPyError"' in result + assert result.count('"PortaPy operation failed"') == 2 assert "dict[str, _Slot]" in result assert "kind: ValueKind = ValueKind.INT" in result @@ -51,5 +54,5 @@ def test_rejects_unexpected_reference_runtime_shape( path = tmp_path / "reference_api.py" path.write_text("import traceback\n", encoding="utf-8") monkeypatch.setattr(normalizer, "PATH", path) - with pytest.raises(RuntimeError, match="traceback formatter normalization"): + with pytest.raises(RuntimeError, match="error capture normalization"): normalizer.main() From 7e8be63bd738580a9dbf87dfe835d875e51fb21f Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:20:39 -0500 Subject: [PATCH 101/450] Disable host-style traceback publication natively --- tools/normalize_full_core_tracebacks.py | 69 ++++++++++++++----------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/tools/normalize_full_core_tracebacks.py b/tools/normalize_full_core_tracebacks.py index f2a0f256..a5d71e22 100644 --- a/tools/normalize_full_core_tracebacks.py +++ b/tools/normalize_full_core_tracebacks.py @@ -1,4 +1,4 @@ -"""Use native-safe string keys for synthetic traceback storage.""" +"""Disable host-style synthetic traceback publication in native builds.""" from __future__ import annotations from pathlib import Path @@ -6,41 +6,50 @@ PATH = Path("src/portapy/core/vm.py") -_REPLACEMENTS = ( - ( - ' self._synthetic_tracebacks: dict[int, "_PyTBProxy"] = {}', - ' self._synthetic_tracebacks: dict[str, "_PyTBProxy"] = {}', - "traceback table annotation", - ), - ( - "self._synthetic_tracebacks.get(id(target), target.__traceback__)", - "self._synthetic_tracebacks.get(str(id(target)), target.__traceback__)", - "traceback attribute lookup", - ), - ( - "prior = self._synthetic_tracebacks.get(id(exc))", - "prior = self._synthetic_tracebacks.get(str(id(exc)))", - "traceback prior lookup", - ), - ( - "self._synthetic_tracebacks[id(exc)] = _PyTBProxy(tb_frame, prior)", - "self._synthetic_tracebacks[str(id(exc))] = _PyTBProxy(tb_frame, prior)", - "traceback storage", - ), +_TABLE = ' self._synthetic_tracebacks: dict[int, "_PyTBProxy"] = {}\n' +_TRACEBACK_LOOKUP = ( + "self._synthetic_tracebacks.get(id(target), target.__traceback__)" ) +_TRACEBACK_BLOCK = ''' if isinstance(exc, BaseException) and not isinstance(exc, PyException): + tb_frame = _PyTBFrameProxy(frame.code, frame.globals, None) + prior = self._synthetic_tracebacks.get(id(exc)) + self._synthetic_tracebacks[id(exc)] = _PyTBProxy(tb_frame, prior) +''' + + +def _replace_once(source: str, old: str, new: str, label: str) -> str: + count = source.count(old) + if count != 1: + raise RuntimeError( + f"native {label}: expected one source form, found {count}" + ) + return source.replace(old, new, 1) def main() -> int: source = PATH.read_text(encoding="utf-8") - for old, new, label in _REPLACEMENTS: - count = source.count(old) - if count != 1: - raise RuntimeError( - f"native {label}: expected one source form, found {count}" - ) - source = source.replace(old, new, 1) + source = _replace_once( + source, + _TABLE, + "", + "traceback table annotation", + ) + source = _replace_once( + source, + _TRACEBACK_LOOKUP, + "None", + "traceback attribute lookup", + ) + source = _replace_once( + source, + _TRACEBACK_BLOCK, + "", + "traceback publication block", + ) + if "_synthetic_tracebacks" in source: + raise RuntimeError("native traceback normalization left stale storage references") PATH.write_text(source, encoding="utf-8") - print("NORMALIZED NATIVE TRACEBACK KEYS", len(_REPLACEMENTS)) + print("DISABLED NATIVE SYNTHETIC TRACEBACK PUBLICATION", 3) return 0 From bc114bd38fd568b0b5d776dd5b002e488d5e725c Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:21:01 -0500 Subject: [PATCH 102/450] Test disabled native traceback publication --- tests/test_normalize_full_core_tracebacks.py | 28 ++++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/test_normalize_full_core_tracebacks.py b/tests/test_normalize_full_core_tracebacks.py index 16268e1a..7bb8e5f0 100644 --- a/tests/test_normalize_full_core_tracebacks.py +++ b/tests/test_normalize_full_core_tracebacks.py @@ -12,15 +12,22 @@ def __init__(self) -> None: self._synthetic_tracebacks: dict[int, "_PyTBProxy"] = {} def read(self, target): - return self._synthetic_tracebacks.get(id(target), target.__traceback__) - - def write(self, exc, tb_frame): - prior = self._synthetic_tracebacks.get(id(exc)) - self._synthetic_tracebacks[id(exc)] = _PyTBProxy(tb_frame, prior) + value = self._synthetic_tracebacks.get(id(target), target.__traceback__) + return value + + def write(self, frame, exc): + try: + pass + except BaseException as exc: + if isinstance(exc, BaseException) and not isinstance(exc, PyException): + tb_frame = _PyTBFrameProxy(frame.code, frame.globals, None) + prior = self._synthetic_tracebacks.get(id(exc)) + self._synthetic_tracebacks[id(exc)] = _PyTBProxy(tb_frame, prior) + return exc ''' -def test_uses_string_keys_for_native_tracebacks( +def test_disables_host_style_native_traceback_storage( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: path = tmp_path / "vm.py" @@ -30,11 +37,10 @@ def test_uses_string_keys_for_native_tracebacks( assert normalizer.main() == 0 result = path.read_text(encoding="utf-8") - assert 'dict[str, "_PyTBProxy"]' in result - assert "get(str(id(target))" in result - assert "get(str(id(exc)))" in result - assert "[str(id(exc))]" in result - assert "[id(exc)]" not in result + assert "_synthetic_tracebacks" not in result + assert "value = None" in result + assert "_PyTBProxy(tb_frame, prior)" not in result + assert "return exc" in result def test_rejects_missing_traceback_storage_shape( From 4b9a57ae4c12f11dde6668762b51a6cb532a73da Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:22:33 -0500 Subject: [PATCH 103/450] Mirror VM traceback block indentation in test --- tests/test_normalize_full_core_tracebacks.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/test_normalize_full_core_tracebacks.py b/tests/test_normalize_full_core_tracebacks.py index 7bb8e5f0..a30167e8 100644 --- a/tests/test_normalize_full_core_tracebacks.py +++ b/tests/test_normalize_full_core_tracebacks.py @@ -16,14 +16,15 @@ def read(self, target): return value def write(self, frame, exc): - try: - pass - except BaseException as exc: - if isinstance(exc, BaseException) and not isinstance(exc, PyException): - tb_frame = _PyTBFrameProxy(frame.code, frame.globals, None) - prior = self._synthetic_tracebacks.get(id(exc)) - self._synthetic_tracebacks[id(exc)] = _PyTBProxy(tb_frame, prior) - return exc + if True: + try: + pass + except BaseException as exc: + if isinstance(exc, BaseException) and not isinstance(exc, PyException): + tb_frame = _PyTBFrameProxy(frame.code, frame.globals, None) + prior = self._synthetic_tracebacks.get(id(exc)) + self._synthetic_tracebacks[id(exc)] = _PyTBProxy(tb_frame, prior) + return exc ''' From 4b75d8ee2aaaf6a1afa586bb81a81c96242d081d Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:24:33 -0500 Subject: [PATCH 104/450] Preserve native conformance crash diagnostics --- tools/run_native_release_conformance.py | 46 +++++++++++++++++++------ 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/tools/run_native_release_conformance.py b/tools/run_native_release_conformance.py index 096619f2..b8685d03 100644 --- a/tools/run_native_release_conformance.py +++ b/tools/run_native_release_conformance.py @@ -26,17 +26,33 @@ ) -def _run(command: list[str], *, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: +def _run( + command: list[str], + *, + cwd: Path | None = None, + check: bool = True, +) -> subprocess.CompletedProcess[str]: return subprocess.run( command, cwd=cwd, - check=True, + check=check, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) +def _write_result(path: Path, result: subprocess.CompletedProcess[str]) -> str: + output = result.stdout.strip() + report = output + if result.returncode != 0: + if report: + report += "\n" + report += f"process-exit={result.returncode}" + path.write_text(report + "\n", encoding="utf-8") + return output + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("library", type=Path) @@ -50,6 +66,7 @@ def main(argv: list[str] | None = None) -> int: output_dir.mkdir(parents=True, exist_ok=True) for source_name, expected in HOSTS: + print(f"native-conformance: {source_name}", flush=True) source = repository / "tests" / source_name executable = output_dir / Path(source_name).stem if os.name == "nt": @@ -68,17 +85,20 @@ def main(argv: list[str] | None = None) -> int: if os.name != "nt": compile_command.append("-ldl") _run(compile_command) - result = _run([str(executable), str(library)]) - output = result.stdout.strip() - (output_dir / f"{Path(source_name).stem}.txt").write_text( - output + "\n", encoding="utf-8" - ) + result = _run([str(executable), str(library)], check=False) + output_path = output_dir / f"{Path(source_name).stem}.txt" + output = _write_result(output_path, result) + if result.returncode != 0: + raise RuntimeError( + f"{source_name} exited with {result.returncode}:\n{output}" + ) if expected not in output.splitlines(): raise RuntimeError( f"{source_name} did not emit {expected!r}:\n{output}" ) print(expected) + print("native-conformance: native_environment_adapter_probe.py", flush=True) adapter = _run( [ sys.executable, @@ -86,11 +106,17 @@ def main(argv: list[str] | None = None) -> int: str(library), ], cwd=repository, + check=False, ) - adapter_output = adapter.stdout.strip() - (output_dir / "native_environment_adapter_probe.txt").write_text( - adapter_output + "\n", encoding="utf-8" + adapter_output = _write_result( + output_dir / "native_environment_adapter_probe.txt", + adapter, ) + if adapter.returncode != 0: + raise RuntimeError( + "native environment adapter exited with " + f"{adapter.returncode}:\n{adapter_output}" + ) if "native-environment-adapter: ok" not in adapter_output.splitlines(): raise RuntimeError(f"native environment adapter failed:\n{adapter_output}") print("native-environment-adapter: ok") From 23975a11189c54f82af7b4f28ec0476ffaf42947 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:25:37 -0500 Subject: [PATCH 105/450] Add native-owned structured error coordinates --- tools/normalize_full_reference_runtime.py | 39 ++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/tools/normalize_full_reference_runtime.py b/tools/normalize_full_reference_runtime.py index ecf6b5bb..ee8aa54c 100644 --- a/tools/normalize_full_reference_runtime.py +++ b/tools/normalize_full_reference_runtime.py @@ -23,6 +23,21 @@ "PortaPy operation failed", "PortaPy operation failed", ) + self._error_line = 0 + self._error_column = 0 + return status + + def _capture_native( + self, + status: Status, + type_name: str, + message: str, + line: int = 0, + column: int = 0, + ) -> Status: + self._last_error = ErrorInfo(status, type_name, message, message) + self._error_line = line + self._error_column = column return status ''' _SLOT = '''@dataclass @@ -171,6 +186,19 @@ def box_bytes(self, value: bytes) -> tuple[Status, int]: return self._capture(Status.TYPE_ERROR, TypeError("value is not str")), b"" return Status.OK, slot.value.encode("utf-8") ''' +_LAST_ERROR_INIT = ''' self._last_error: ErrorInfo | None = None + self._closed = False +''' +_NATIVE_LAST_ERROR_INIT = ''' self._last_error: ErrorInfo | None = None + self._error_line = 0 + self._error_column = 0 + self._closed = False +''' +_CLEAR_ERROR = " self._last_error = None\n" +_NATIVE_CLEAR_ERROR = ''' self._last_error = None + self._error_line = 0 + self._error_column = 0 +''' _VALUES_ANNOTATION = "self._values: dict[int, _Slot] = {}" _NATIVE_VALUES_ANNOTATION = "self._values: dict[str, _Slot] = {}" _HANDLE_SUBSCRIPT = "self._values[handle]" @@ -210,10 +238,18 @@ def main() -> int: (_AS_INT_METHOD, _NATIVE_AS_INT_METHOD, "integer conversion"), (_AS_FLOAT_METHOD, _NATIVE_AS_FLOAT_METHOD, "float conversion"), (_AS_UTF8_METHOD, _NATIVE_AS_UTF8_METHOD, "UTF-8 conversion"), + (_LAST_ERROR_INIT, _NATIVE_LAST_ERROR_INIT, "error coordinate initialization"), (_VALUES_ANNOTATION, _NATIVE_VALUES_ANNOTATION, "value-table annotation"), ) for old, new, label in replacements: source = _replace_exact(source, old, new, label=label, expected=1) + source = _replace_exact( + source, + _CLEAR_ERROR, + _NATIVE_CLEAR_ERROR, + label="error coordinate reset", + expected=3, + ) source = _replace_exact( source, _HANDLE_SUBSCRIPT, @@ -236,7 +272,8 @@ def main() -> int: expected=1, ) PATH.write_text(source, encoding="utf-8") - print("NORMALIZED NATIVE REFERENCE ERROR CAPTURE", 1) + print("NORMALIZED NATIVE REFERENCE ERROR CAPTURE", 2) + print("NORMALIZED NATIVE ERROR COORDINATES", 4) print("NORMALIZED NATIVE VALUE HANDLE KEYS", 11) print("NORMALIZED NATIVE VALUE KIND SLOTS", 9) return 0 From 8735c85536d967de7c17eb704640800cea10039c Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:26:06 -0500 Subject: [PATCH 106/450] Test native structured error coordinates --- tests/test_normalize_full_reference_runtime.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_normalize_full_reference_runtime.py b/tests/test_normalize_full_reference_runtime.py index c5a673b7..016e8a46 100644 --- a/tests/test_normalize_full_reference_runtime.py +++ b/tests/test_normalize_full_reference_runtime.py @@ -27,6 +27,13 @@ def test_normalizes_real_reference_runtime( assert '"PortaPyError"' in result assert result.count('"PortaPy operation failed"') == 2 + assert "def _capture_native(" in result + assert "ErrorInfo(status, type_name, message, message)" in result + assert "self._error_line = line" in result + assert "self._error_column = column" in result + assert result.count("self._error_line = 0") >= 4 + assert result.count("self._error_column = 0") >= 4 + assert "dict[str, _Slot]" in result assert "kind: ValueKind = ValueKind.INT" in result assert "self._values[str(handle)] = _Slot(value, kind)" in result From 7bda204ced4c551023ca41185e9008234b223683 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:27:21 -0500 Subject: [PATCH 107/450] Add precise native structured error normalization --- tools/normalize_full_reference_errors.py | 206 +++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 tools/normalize_full_reference_errors.py diff --git a/tools/normalize_full_reference_errors.py b/tools/normalize_full_reference_errors.py new file mode 100644 index 00000000..09c57670 --- /dev/null +++ b/tools/normalize_full_reference_errors.py @@ -0,0 +1,206 @@ +"""Install native-safe structured error reporting in the full ABI entry.""" +from __future__ import annotations + +import ast +from pathlib import Path + + +PATH = Path("src/portapy/native_full_reference_entry.py") + +_LOCATION_HELPER = ''' +def _native_error_location(source: str) -> tuple[int, int]: + line = 1 + column = 1 + index = 0 + size = len(source) + while index < size: + char = source[index] + if char == "\n": + line += 1 + column = 1 + index += 1 + continue + operator_size = 0 + if char == "%": + operator_size = 1 + elif char == "/": + operator_size = 1 + if index + 1 < size and source[index + 1] == "/": + operator_size = 2 + if operator_size: + lookahead = index + operator_size + while lookahead < size and ( + source[lookahead] == " " or source[lookahead] == "\t" + ): + lookahead += 1 + if lookahead < size and source[lookahead] == "0": + return line, column + index += 1 + column += 1 + return 1, 1 +''' + +_EXEC_SOURCE = ''' +instance = _runtime(runtime) +if instance is None: + return _set_status(Status.INVALID_HANDLE) +if source_size < 0: + return _set_status(Status.INVALID_ARGUMENT) +source_text = source[0:source_size] +status = instance.exec_utf8(source_text) +if status is Status.RUNTIME_ERROR: + line, column = _native_error_location(source_text) + instance._capture_native( + status, + "RuntimeError", + "PortaPy source execution failed", + line, + column, + ) +elif status is Status.COMPILE_ERROR: + line, column = _native_error_location(source_text) + instance._capture_native( + status, + "SyntaxError", + "PortaPy source compilation failed", + line, + column, + ) +return _set_status(status) +''' + +_EVAL_SOURCE = ''' +instance = _runtime(runtime) +if instance is None: + _set_status(Status.INVALID_HANDLE) + return 0 +if source_size < 0: + _set_status(Status.INVALID_ARGUMENT) + return 0 +source_text = source[0:source_size] +status, value = instance.eval_utf8(source_text) +if status is Status.RUNTIME_ERROR: + line, column = _native_error_location(source_text) + instance._capture_native( + status, + "RuntimeError", + "PortaPy source evaluation failed", + line, + column, + ) +elif status is Status.COMPILE_ERROR: + line, column = _native_error_location(source_text) + instance._capture_native( + status, + "SyntaxError", + "PortaPy source compilation failed", + line, + column, + ) +_set_status(status) +return value +''' + +_VALIDATE_UTF8_SOURCE = ''' +instance = _runtime(runtime) +if instance is None: + return _set_status(Status.INVALID_HANDLE) +status, raw = instance.unbox(value) +if status is not Status.OK: + return _set_status(status) +try: + raw.decode("utf-8") +except UnicodeDecodeError: + status = instance._capture_native( + Status.TYPE_ERROR, + "UnicodeDecodeError", + "invalid UTF-8", + 0, + 1, + ) + return _set_status(status) +return _set_status(Status.OK) +''' + +_ERROR_LINE_SOURCE = ''' +instance = _runtime(runtime) +if instance is None: + _set_status(Status.INVALID_HANDLE) + return 0 +if instance.last_error() is None: + _set_status(Status.NOT_FOUND) + return 0 +_set_status(Status.OK) +return instance._error_line +''' + +_ERROR_COLUMN_SOURCE = ''' +instance = _runtime(runtime) +if instance is None: + _set_status(Status.INVALID_HANDLE) + return 0 +if instance.last_error() is None: + _set_status(Status.NOT_FOUND) + return 0 +_set_status(Status.OK) +return instance._error_column +''' + +_REPLACEMENTS = { + "_portapy_exec_span_impl": _EXEC_SOURCE, + "_portapy_eval_span_impl": _EVAL_SOURCE, + "_portapy_value_validate_utf8_impl": _VALIDATE_UTF8_SOURCE, + "_portapy_error_line_impl": _ERROR_LINE_SOURCE, + "_portapy_error_column_impl": _ERROR_COLUMN_SOURCE, +} + + +class _Rewrite(ast.NodeTransformer): + def __init__(self) -> None: + self.replaced: set[str] = set() + + def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST: + replacement = _REPLACEMENTS.get(node.name) + if replacement is None: + return self.generic_visit(node) + node.body = ast.parse(replacement).body + self.replaced.add(node.name) + return node + + +def main() -> int: + module = ast.parse(PATH.read_text(encoding="utf-8")) + if any( + isinstance(node, ast.FunctionDef) and node.name == "_native_error_location" + for node in module.body + ): + raise RuntimeError("native error helpers are already installed") + + rewriter = _Rewrite() + module = rewriter.visit(module) + missing = sorted(set(_REPLACEMENTS) - rewriter.replaced) + if missing: + raise RuntimeError(f"native structured error functions missing: {missing}") + module.body.extend(ast.parse(_LOCATION_HELPER).body) + ast.fix_missing_locations(module) + source = ast.unparse(module) + "\n" + PATH.write_text(source, encoding="utf-8") + + verified = ast.parse(source) + text = ast.unparse(verified) + required = ( + '"UnicodeDecodeError"', + "instance._error_line", + "instance._error_column", + "_native_error_location(source_text)", + "instance._capture_native(", + ) + absent = [marker for marker in required if marker not in text] + if absent: + raise RuntimeError(f"native structured error validation failed: {absent}") + print("NORMALIZED NATIVE STRUCTURED ERRORS", len(_REPLACEMENTS)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From e686e1d271e6298c35780d86fafa8c7a7f5f9fca Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:27:48 -0500 Subject: [PATCH 108/450] Wire native structured errors into full build --- tools/normalize_full_core_validation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/normalize_full_core_validation.py b/tools/normalize_full_core_validation.py index 2fa188bd..d1fc5188 100644 --- a/tools/normalize_full_core_validation.py +++ b/tools/normalize_full_core_validation.py @@ -27,6 +27,9 @@ from tools.normalize_full_reference_abi_helpers import ( main as normalize_reference_abi_helpers, ) +from tools.normalize_full_reference_errors import ( + main as normalize_reference_errors, +) from tools.normalize_full_reference_runtime import ( main as normalize_reference_runtime, ) @@ -77,6 +80,7 @@ def main() -> int: materialize_reference_entry() normalize_reference_abi_helpers() + normalize_reference_errors() normalize_native_parser() normalize_calls_closures() normalize_keyword_calls() From 06a7d6266f3467cf50d716a8366a85edad261865 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:28:12 -0500 Subject: [PATCH 109/450] Test native structured error normalization --- tests/test_normalize_full_reference_errors.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 tests/test_normalize_full_reference_errors.py diff --git a/tests/test_normalize_full_reference_errors.py b/tests/test_normalize_full_reference_errors.py new file mode 100644 index 00000000..43c202a4 --- /dev/null +++ b/tests/test_normalize_full_reference_errors.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import materialize_full_reference_entry as materializer +from tools import normalize_full_reference_abi_helpers as abi_normalizer +from tools import normalize_full_reference_errors as error_normalizer + + +def _function(module: ast.Module, name: str) -> str: + node = next( + item + for item in module.body + if isinstance(item, ast.FunctionDef) and item.name == name + ) + return ast.unparse(node) + + +def test_installs_native_structured_error_paths( + tmp_path: Path, monkeypatch, +) -> None: + output = tmp_path / "native_full_reference_entry.py" + monkeypatch.setattr(materializer, "OUTPUT", output) + monkeypatch.setattr(abi_normalizer, "PATH", output) + monkeypatch.setattr(error_normalizer, "PATH", output) + + assert materializer.main() == 0 + assert abi_normalizer.main() == 0 + assert error_normalizer.main() == 0 + + module = ast.parse(output.read_text(encoding="utf-8")) + validate = _function(module, "_portapy_value_validate_utf8_impl") + assert '"UnicodeDecodeError"' in validate + assert "instance._capture_native(" in validate + + execute = _function(module, "_portapy_exec_span_impl") + evaluate = _function(module, "_portapy_eval_span_impl") + assert "_native_error_location(source_text)" in execute + assert "_native_error_location(source_text)" in evaluate + assert '"RuntimeError"' in execute + assert '"SyntaxError"' in evaluate + + line = _function(module, "_portapy_error_line_impl") + column = _function(module, "_portapy_error_column_impl") + assert "return instance._error_line" in line + assert "return instance._error_column" in column + + +def test_native_error_location_finds_division_by_zero() -> None: + namespace: dict[str, object] = {} + exec(error_normalizer._LOCATION_HELPER, namespace) + locate = namespace["_native_error_location"] + + assert locate("safe = 1\nbroken = 5 // 0") == (2, 12) + assert locate("value = 9 % 0") == (1, 9) + assert locate("value = 4") == (1, 1) From 405df88cfe2d88c4779cf2f99ea02bdf8a4a602c Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:28:38 -0500 Subject: [PATCH 110/450] Correct native error column expectation --- tests/test_normalize_full_reference_errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_normalize_full_reference_errors.py b/tests/test_normalize_full_reference_errors.py index 43c202a4..b45921fb 100644 --- a/tests/test_normalize_full_reference_errors.py +++ b/tests/test_normalize_full_reference_errors.py @@ -53,5 +53,5 @@ def test_native_error_location_finds_division_by_zero() -> None: locate = namespace["_native_error_location"] assert locate("safe = 1\nbroken = 5 // 0") == (2, 12) - assert locate("value = 9 % 0") == (1, 9) + assert locate("value = 9 % 0") == (1, 11) assert locate("value = 4") == (1, 1) From 11b132fe4bae9f3b9628092fe4a8702521ad6956 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:40:40 -0500 Subject: [PATCH 111/450] Fix structured-error helper escaping --- tools/normalize_full_reference_errors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/normalize_full_reference_errors.py b/tools/normalize_full_reference_errors.py index 09c57670..22a68863 100644 --- a/tools/normalize_full_reference_errors.py +++ b/tools/normalize_full_reference_errors.py @@ -15,7 +15,7 @@ def _native_error_location(source: str) -> tuple[int, int]: size = len(source) while index < size: char = source[index] - if char == "\n": + if char == "\\n": line += 1 column = 1 index += 1 @@ -30,7 +30,7 @@ def _native_error_location(source: str) -> tuple[int, int]: if operator_size: lookahead = index + operator_size while lookahead < size and ( - source[lookahead] == " " or source[lookahead] == "\t" + source[lookahead] == " " or source[lookahead] == "\\t" ): lookahead += 1 if lookahead < size and source[lookahead] == "0": From 4bf3e1064ed00bf98c0d3e86390b1dc0a19438b0 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:44:29 -0500 Subject: [PATCH 112/450] Make structured-error validation quote-independent --- tools/normalize_full_reference_errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/normalize_full_reference_errors.py b/tools/normalize_full_reference_errors.py index 22a68863..ffd15e85 100644 --- a/tools/normalize_full_reference_errors.py +++ b/tools/normalize_full_reference_errors.py @@ -189,7 +189,7 @@ def main() -> int: verified = ast.parse(source) text = ast.unparse(verified) required = ( - '"UnicodeDecodeError"', + "UnicodeDecodeError", "instance._error_line", "instance._error_column", "_native_error_location(source_text)", From bfe03b32a27a581b7aa45c666ad595c0a46f0d29 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:44:45 -0500 Subject: [PATCH 113/450] Make structured-error tests quote-independent --- tests/test_normalize_full_reference_errors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_normalize_full_reference_errors.py b/tests/test_normalize_full_reference_errors.py index b45921fb..98671261 100644 --- a/tests/test_normalize_full_reference_errors.py +++ b/tests/test_normalize_full_reference_errors.py @@ -31,15 +31,15 @@ def test_installs_native_structured_error_paths( module = ast.parse(output.read_text(encoding="utf-8")) validate = _function(module, "_portapy_value_validate_utf8_impl") - assert '"UnicodeDecodeError"' in validate + assert "UnicodeDecodeError" in validate assert "instance._capture_native(" in validate execute = _function(module, "_portapy_exec_span_impl") evaluate = _function(module, "_portapy_eval_span_impl") assert "_native_error_location(source_text)" in execute assert "_native_error_location(source_text)" in evaluate - assert '"RuntimeError"' in execute - assert '"SyntaxError"' in evaluate + assert "RuntimeError" in execute + assert "SyntaxError" in evaluate line = _function(module, "_portapy_error_line_impl") column = _function(module, "_portapy_error_column_impl") From ce8f8651fac486530b3604a99ded117722025100 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:54:03 -0500 Subject: [PATCH 114/450] Normalize native floats through IEEE-754 bit payloads --- tools/normalize_full_reference_float_bits.py | 110 +++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 tools/normalize_full_reference_float_bits.py diff --git a/tools/normalize_full_reference_float_bits.py b/tools/normalize_full_reference_float_bits.py new file mode 100644 index 00000000..24482b18 --- /dev/null +++ b/tools/normalize_full_reference_float_bits.py @@ -0,0 +1,110 @@ +"""Use integer bit payloads at the native binary64 ABI boundary. + +The generic Runtime handle table stores Python object-shaped values through general +purpose registers. Native ``float`` parameters use XMM registers, so passing them +directly into the generic ``_store`` method loses both payload and kind metadata. +The public C ABI still accepts and returns ordinary ``double`` values; the assembly +adapter moves their IEEE-754 bits into integer registers before entering these +Python-authored implementation functions. +""" +from __future__ import annotations + +import ast +from pathlib import Path + + +PATH = Path("src/portapy/native_full_reference_entry.py") + +_FROM_BITS_SOURCE = ''' +instance = _runtime(runtime) +if instance is None: + _set_status(Status.INVALID_HANDLE) + return 0 +result = instance._store(bits, ValueKind.FLOAT) +_set_status(Status.OK) +return result +''' + +_AS_BITS_SOURCE = ''' +instance = _runtime(runtime) +if instance is None: + _set_status(Status.INVALID_HANDLE) + return 0 +status, kind = instance.value_kind(value) +if status is not Status.OK: + _set_status(status) + return 0 +if kind is not ValueKind.FLOAT: + _set_status(Status.TYPE_ERROR) + return 0 +status, result = instance.unbox(value) +_set_status(status) +return result +''' + + +class _Rewrite(ast.NodeTransformer): + def __init__(self) -> None: + self.replaced: set[str] = set() + + def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST: + self.generic_visit(node) + if node.name == "_portapy_value_from_f64_impl": + if len(node.args.args) != 2: + raise RuntimeError("native float constructor has unexpected signature") + node.name = "_portapy_value_from_f64_bits_impl" + node.args.args[1].arg = "bits" + node.args.args[1].annotation = ast.Name(id="int", ctx=ast.Load()) + node.returns = ast.Name(id="int", ctx=ast.Load()) + node.body = ast.parse(_FROM_BITS_SOURCE).body + self.replaced.add("from") + elif node.name == "_portapy_value_as_f64_impl": + if len(node.args.args) != 2: + raise RuntimeError("native float conversion has unexpected signature") + node.name = "_portapy_value_as_f64_bits_impl" + node.returns = ast.Name(id="int", ctx=ast.Load()) + node.body = ast.parse(_AS_BITS_SOURCE).body + self.replaced.add("as") + return node + + +def main() -> int: + module = ast.parse(PATH.read_text(encoding="utf-8")) + rewriter = _Rewrite() + module = rewriter.visit(module) + if rewriter.replaced != {"from", "as"}: + raise RuntimeError( + "native float-bit normalization expected constructor and conversion; " + f"replaced={sorted(rewriter.replaced)}" + ) + ast.fix_missing_locations(module) + source = ast.unparse(module) + "\n" + PATH.write_text(source, encoding="utf-8") + + verified = ast.parse(source) + definitions = { + node.name: node + for node in verified.body + if isinstance(node, ast.FunctionDef) + } + stale = { + "_portapy_value_from_f64_impl", + "_portapy_value_as_f64_impl", + } & definitions.keys() + from_text = ast.unparse(definitions["_portapy_value_from_f64_bits_impl"]) + as_text = ast.unparse(definitions["_portapy_value_as_f64_bits_impl"]) + ready = ( + not stale + and "instance._store(bits, ValueKind.FLOAT)" in from_text + and "instance.value_kind(value)" in as_text + and "kind is not ValueKind.FLOAT" in as_text + and "instance.unbox(value)" in as_text + ) + if not ready: + raise RuntimeError("native float-bit normalization validation failed") + print("NORMALIZED NATIVE FLOAT BIT PAYLOADS", 2) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 61b0625fdbe934aab52bc737d845084fbfbbd622 Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:54:28 -0500 Subject: [PATCH 115/450] Wire native float bit normalization into full pipeline --- tools/normalize_full_core_validation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/normalize_full_core_validation.py b/tools/normalize_full_core_validation.py index d1fc5188..e8cb8195 100644 --- a/tools/normalize_full_core_validation.py +++ b/tools/normalize_full_core_validation.py @@ -30,6 +30,9 @@ from tools.normalize_full_reference_errors import ( main as normalize_reference_errors, ) +from tools.normalize_full_reference_float_bits import ( + main as normalize_reference_float_bits, +) from tools.normalize_full_reference_runtime import ( main as normalize_reference_runtime, ) @@ -80,6 +83,7 @@ def main() -> int: materialize_reference_entry() normalize_reference_abi_helpers() + normalize_reference_float_bits() normalize_reference_errors() normalize_native_parser() normalize_calls_closures() From 221b6ad8756fd5802b6e955ba08d9e8ec1c4020c Mon Sep 17 00:00:00 2001 From: deltathedumb <133447617+deltathedumb@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:55:01 -0500 Subject: [PATCH 116/450] Bridge native floats through integer bit payloads --- tools/nasm_direct_float_abi.py | 36 +++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/tools/nasm_direct_float_abi.py b/tools/nasm_direct_float_abi.py index 275723b4..81e74ebb 100644 --- a/tools/nasm_direct_float_abi.py +++ b/tools/nasm_direct_float_abi.py @@ -1,8 +1,10 @@ -"""Append C-ABI wrappers for direct native binary64 Runtime functions. +"""Append C-ABI wrappers for native binary64 Runtime functions. -Unlike the incremental core, the full Runtime's compiled Python entry accepts -and returns real ``float`` values through XMM registers. These wrappers preserve -out-parameters and status handling without converting through integer payloads. +The public ABI accepts and returns ordinary C ``double`` values. The compiled +Python Runtime stores their IEEE-754 payloads as integers tagged ``FLOAT`` so the +generic handle table never mixes XMM-register parameters with object-register +parameters. These wrappers move the bits between the platform C ABI and the +integer implementation functions without changing a single bit. """ from __future__ import annotations @@ -14,8 +16,8 @@ _LABEL_RE = re.compile(r"^(?P