diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cd68461..10b520f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,28 +1,181 @@ name: CI + on: push: + branches: [main] pull_request: + branches: [main] workflow_dispatch: + +concurrency: + group: ci-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + permissions: contents: read + jobs: - bootstrap: + hosted: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: '3.12' - run: python -m pip install -e '.[test]' - - name: Run hosted tests + - name: Run complete hosted suite + shell: bash + run: | + set -o pipefail + python -m pytest -x -vv 2>&1 | tee hosted-pytest.log + - name: Check public C example + run: cc -std=c11 -Wall -Wextra -Werror -Iinclude -fsyntax-only examples/embed.c + - uses: actions/upload-artifact@v4 + if: always() + with: + name: portapy-hosted-pytest + path: hosted-pytest.log + + audit: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install verified asmpython + run: | + python -m pip install --no-cache-dir --force-reinstall \ + 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' + - name: Audit full-core imports and source positions + shell: bash + run: | + mkdir -p dist + python tools/normalize_full_core_probe.py + python tools/normalize_full_core_lambdas.py + python -m asmpython._compiler.import_audit \ + src/portapy/native_full_core_probe.py \ + > dist/full-core-import-audit.txt 2>&1 + python tools/asmpython_parse_audit.py \ + src/portapy/core/frontend.py \ + src/portapy/core/vm.py \ + > dist/full-core-parse-audit.txt 2>&1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: full-core-import-audit + path: | + dist/full-core-import-audit.txt + dist/full-core-parse-audit.txt + + native: + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-24.04 + target: linux + - os: windows-2025 + target: windows + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install Linux dependencies + if: matrix.target == 'linux' + shell: bash + run: | + sudo apt-get update + sudo apt-get install --yes nasm gcc binutils + python -m pip install -e '.[test]' + python -m pip install --no-cache-dir --force-reinstall \ + 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' + - name: Install Windows dependencies + if: matrix.target == 'windows' + shell: powershell + run: | + & .\tools\install_windows_toolchain.ps1 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + python -m pip install -e '.[test]' + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & .\tools\install_pinned_asmpython.ps1 + - name: Build Linux standalone library + if: matrix.target == 'linux' shell: bash run: | mkdir -p dist set -o pipefail - PYTHONPATH="$PWD" python -m pytest -q 2>&1 | tee dist/pytest.log - - run: cc -std=c11 -Wall -Wextra -Werror -Iinclude -fsyntax-only examples/embed.c + python tools/build_native_typed.py \ + --target linux \ + --output dist/libportapy.so \ + --work-dir dist/build-verify-linux \ + 2>&1 | tee dist/build-linux.log + - name: Build Windows standalone library + if: matrix.target == 'windows' + shell: powershell + run: | + New-Item -ItemType Directory -Force dist | Out-Null + $output = & python tools/build_native_typed.py ` + --target windows ` + --output dist/portapy.dll ` + --work-dir dist/build-verify-windows ` + 2>&1 + $buildStatus = $LASTEXITCODE + $output | Tee-Object -FilePath dist/build-windows.log + if ($buildStatus -ne 0) { exit $buildStatus } + - name: Verify Linux standalone library + if: matrix.target == 'linux' + shell: bash + run: | + if readelf -dW dist/libportapy.so | grep -Ei 'libpython|python3'; then + echo 'CPython linkage is forbidden' + exit 1 + fi + if readelf -dW dist/libportapy.so | grep -q TEXTREL; then + echo 'text relocations are forbidden' + exit 1 + fi + python tools/run_native_release_conformance.py \ + dist/libportapy.so \ + --output-dir dist/conformance-linux + - name: Verify Windows standalone library + if: matrix.target == 'windows' + shell: powershell + run: | + $imports = (& objdump -p dist/portapy.dll | Out-String) + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if ($imports -match 'DLL Name:\s*(python|libpython)[^\s]*\.dll') { + throw 'CPython linkage is forbidden' + } + python tools/run_native_release_conformance.py ` + dist/portapy.dll ` + --compiler gcc ` + --output-dir dist/conformance-windows + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + - name: Verify direct CSharp PInvoke + if: matrix.target == 'windows' + shell: powershell + run: | + dotnet build examples/csharp/PortaPyExample.csproj ` + --configuration Release ` + --output dist/csharp + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Copy-Item dist/portapy.dll dist/csharp/portapy.dll + Push-Location dist/csharp + try { + $output = @(dotnet PortaPyExample.dll) + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } finally { + Pop-Location + } + $output | Out-File -Encoding utf8 dist/csharp-output.txt + if ($output[-1] -ne 'Hello, world!') { + throw "unexpected C# output: $($output -join [Environment]::NewLine)" + } - uses: actions/upload-artifact@v4 if: always() with: - name: hosted-ci + name: portapy-ci-${{ matrix.target }} path: dist diff --git a/.github/workflows/csharp-ffi.yml b/.github/workflows/csharp-ffi.yml deleted file mode 100644 index fe83fecb..00000000 --- a/.github/workflows/csharp-ffi.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: CSharp direct FFI - -on: - pull_request: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - -jobs: - windows: - runs-on: windows-2025 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install native toolchain and asmpython - shell: powershell - run: | - choco install nasm mingw -y --no-progress - python -m pip install --no-cache-dir --force-reinstall ` - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - - name: Build portapy.dll - shell: powershell - run: | - python tools/build_native_typed.py ` - --target windows ` - --output dist/portapy.dll ` - --work-dir dist/build-csharp-windows - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - name: Build and run direct PInvoke example - shell: powershell - run: | - dotnet build examples/csharp/PortaPyExample.csproj ` - --configuration Release ` - --output dist/csharp - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - Copy-Item dist/portapy.dll dist/csharp/portapy.dll - Push-Location dist/csharp - try { - $output = @(dotnet PortaPyExample.dll) - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - } finally { - Pop-Location - } - $output | Out-File -Encoding utf8 dist/csharp-output.txt - if ($output[-1] -ne 'Hello, world!') { - throw "unexpected C# output: $($output -join [Environment]::NewLine)" - } - - uses: actions/upload-artifact@v4 - if: always() - with: - name: csharp-direct-ffi - path: dist diff --git a/.github/workflows/full-core-import-audit.yml b/.github/workflows/full-core-import-audit.yml deleted file mode 100644 index d62bce62..00000000 --- a/.github/workflows/full-core-import-audit.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Full-core import audit - -on: - pull_request: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - pull-requests: write - -jobs: - audit: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install verified asmpython - run: | - python -m pip install --no-cache-dir --force-reinstall \ - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - - name: Normalize probe source - run: | - python tools/normalize_full_core_probe.py - python tools/normalize_full_core_lambdas.py - - name: Audit full-core project import graph - id: import_audit - continue-on-error: true - run: | - mkdir -p dist - python -m asmpython._compiler.import_audit \ - src/portapy/native_full_core_probe.py \ - > dist/full-core-import-audit.txt 2>&1 - - name: Audit source positions - id: parse_audit - continue-on-error: true - run: | - python tools/asmpython_parse_audit.py \ - src/portapy/core/frontend.py \ - src/portapy/core/vm.py \ - > dist/full-core-parse-audit.txt 2>&1 - - name: Show audits - if: always() - run: | - cat dist/full-core-import-audit.txt - cat dist/full-core-parse-audit.txt - - name: Report failed audits - if: (steps.import_audit.outcome != 'success' || steps.parse_audit.outcome != 'success') && github.event_name == 'pull_request' - env: - GH_TOKEN: ${{ github.token }} - run: | - { - echo '' - echo '### Full-core import and parser audit' - echo - echo '```text' - cat dist/full-core-import-audit.txt - echo - cat dist/full-core-parse-audit.txt - echo '```' - } > /tmp/portapy-full-core-import-audit.md - gh pr comment "${{ github.event.pull_request.number }}" \ - --body-file /tmp/portapy-full-core-import-audit.md - - uses: actions/upload-artifact@v4 - if: always() - with: - name: full-core-import-audit - path: | - dist/full-core-import-audit.txt - dist/full-core-parse-audit.txt - - name: Require clean audits - if: steps.import_audit.outcome != 'success' || steps.parse_audit.outcome != 'success' - run: exit 1 diff --git a/.github/workflows/native-environment-adapter.yml b/.github/workflows/native-environment-adapter.yml deleted file mode 100644 index 2f5bc87f..00000000 --- a/.github/workflows/native-environment-adapter.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Native environment adapter - -on: - pull_request: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - -jobs: - linux: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install package, toolchain, and asmpython - run: | - sudo apt-get update - sudo apt-get install --yes nasm gcc binutils - python -m pip install -e '.[test]' - python -m pip install --no-cache-dir --force-reinstall \ - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - - name: Build native library - run: | - python tools/build_native_typed.py \ - --target linux \ - --output dist/libportapy.so \ - --work-dir dist/build-linux - - name: Run Python facade against native library - run: | - set +e - python tests/native_environment_adapter_probe.py dist/libportapy.so \ - 2>&1 | tee dist/environment-adapter-output.txt - status=${PIPESTATUS[0]} - set -e - test "$status" -eq 0 - grep -qx 'native-environment-adapter: ok' dist/environment-adapter-output.txt - - uses: actions/upload-artifact@v4 - if: always() - with: - name: linux-native-environment-adapter - path: dist - - windows: - runs-on: windows-2025 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install package, toolchain, and asmpython - shell: powershell - run: | - choco install nasm mingw -y --no-progress - python -m pip install -e '.[test]' - python -m pip install --no-cache-dir --force-reinstall ` - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - - name: Build native library - shell: powershell - run: | - python tools/build_native_typed.py ` - --target windows ` - --output dist/portapy.dll ` - --work-dir dist/build-windows - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - name: Run Python facade against native library - shell: powershell - run: | - $output = & python tests/native_environment_adapter_probe.py dist/portapy.dll 2>&1 - $status = $LASTEXITCODE - $output | Tee-Object -FilePath dist/environment-adapter-output.txt - if ($status -ne 0) { exit $status } - if (($output -join "`n") -ne 'native-environment-adapter: ok') { - throw "unexpected adapter output: $output" - } - - uses: actions/upload-artifact@v4 - if: always() - with: - name: windows-native-environment-adapter - path: dist diff --git a/.github/workflows/native-full-core-probe.yml b/.github/workflows/native-full-core-probe.yml index 4115e262..5d20b3e1 100644 --- a/.github/workflows/native-full-core-probe.yml +++ b/.github/workflows/native-full-core-probe.yml @@ -29,12 +29,23 @@ jobs: python -m pip install --no-cache-dir --force-reinstall \ 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - name: Normalize probe source + shell: bash run: | - python tools/normalize_full_core_probe.py - python tools/normalize_full_core_lambdas.py - python tools/normalize_full_core_native_semantics.py - python tools/normalize_full_core_opcode_maps.py - python tools/normalize_full_core_validation.py + mkdir -p dist + set +e + python -m tools.normalize_full_core_validation \ + > dist/linux-full-core-normalization.txt 2>&1 + status=$? + set -e + if [ "$status" -ne 0 ]; then + if [ -f dist/full-core-normalization-error.txt ]; then + cat dist/full-core-normalization-error.txt + else + cat dist/linux-full-core-normalization.txt + fi + exit "$status" + fi + cat dist/linux-full-core-normalization.txt mkdir -p dist/debug-tree python - <<'PY' from pathlib import Path @@ -87,6 +98,7 @@ jobs: MAP gcc -shared dist/libportapy_full_core_probe.o \ -Wl,--version-script=dist/portapy_full_core_probe.map \ + -lm \ -o dist/libportapy_full_core_probe.so - name: Execute full core from external C host run: | @@ -129,13 +141,17 @@ jobs: run: | { echo '' - echo '### Linux full-core compiler diagnostic' + echo '### Linux full-core diagnostic' echo echo '```text' - if [ -f dist/linux-full-core-build.log ]; then + if [ -f dist/full-core-normalization-error.txt ]; then + cat dist/full-core-normalization-error.txt + elif [ -f dist/linux-full-core-build.log ]; then tail -n 180 dist/linux-full-core-build.log + elif [ -f dist/linux-full-core-normalization.txt ]; then + cat dist/linux-full-core-normalization.txt else - echo 'No compiler log was produced.' + echo 'No diagnostic log was produced.' fi echo '```' } > /tmp/portapy-full-core-linux.md @@ -164,36 +180,50 @@ jobs: - name: Normalize probe source shell: powershell run: | - python tools/normalize_full_core_probe.py - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - python tools/normalize_full_core_lambdas.py - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - python tools/normalize_full_core_native_semantics.py - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - python tools/normalize_full_core_opcode_maps.py - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - python tools/normalize_full_core_validation.py - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + New-Item -ItemType Directory -Force dist | Out-Null + $normalizationLog = Join-Path $PWD 'dist/windows-full-core-normalization.txt' + python -m tools.normalize_full_core_validation *> $normalizationLog + $status = $LASTEXITCODE + if ($status -ne 0) { + if (Test-Path dist/full-core-normalization-error.txt) { + Get-Content dist/full-core-normalization-error.txt + } else { + Get-Content $normalizationLog + } + exit $status + } + Get-Content $normalizationLog - name: Compile full frontend and VM probe shell: powershell run: | New-Item -ItemType Directory -Force dist | Out-Null - python tools/run_full_core_asmpython.py build src/portapy/native_full_core_probe.py ` - --target windows --type library --backend legacy ` - --no-pyinbin-fallback --keep-assembly ` - -o dist/portapy_full_core_probe.dll ` - *>&1 | Tee-Object -FilePath dist/windows-full-core-build.log - $compilerExit = $LASTEXITCODE + $oldErrorPreference = $ErrorActionPreference + $oldNativeErrorPreference = $PSNativeCommandUseErrorActionPreference + $ErrorActionPreference = 'Continue' + $PSNativeCommandUseErrorActionPreference = $false + try { + python tools/run_full_core_asmpython.py build src/portapy/native_full_core_probe.py ` + --target windows --type library --backend legacy ` + --no-pyinbin-fallback --keep-assembly ` + -o dist/portapy_full_core_probe.dll ` + *>&1 | Tee-Object -FilePath dist/windows-full-core-build.log + $compilerExit = $LASTEXITCODE + } finally { + $ErrorActionPreference = $oldErrorPreference + $PSNativeCommandUseErrorActionPreference = $oldNativeErrorPreference + } if (-not (Test-Path dist/portapy_full_core_probe.asm)) { exit $compilerExit } python tools/nasm_module_init.py dist/portapy_full_core_probe.asm ` --target windows --public-symbol portapy_library_initialize + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python tools/nasm_exports.py dist/portapy_full_core_probe.asm ` --export portapy_library_initialize ` --export portapy_abi_version ` --export portapy_full_core_parse_probe ` --export portapy_full_core_probe + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } nasm -f win64 -w-label-redef-late ` dist/portapy_full_core_probe.asm ` -o dist/portapy_full_core_probe.obj @@ -255,19 +285,23 @@ jobs: run: | $lines = @( '', - '### Windows full-core compiler diagnostic', + '### Windows full-core diagnostic', '', '```text' ) - if (Test-Path dist/windows-full-core-build.log) { - $lines += Get-Content dist/windows-full-core-build.log -Tail 180 + if (Test-Path dist/full-core-normalization-error.txt) { + $lines += Get-Content dist/full-core-normalization-error.txt + } elseif (Test-Path dist/windows-full-core-build.log) { + $lines += Get-Content dist/windows-full-core-build.log | Select-Object -Last 180 + } elseif (Test-Path dist/windows-full-core-normalization.txt) { + $lines += Get-Content dist/windows-full-core-normalization.txt } else { - $lines += 'No compiler log was produced.' + $lines += 'No diagnostic log was produced.' } $lines += '```' - $path = "$env:RUNNER_TEMP\portapy-full-core-windows.md" - $lines | Set-Content -Encoding utf8 $path - gh pr comment "${{ github.event.pull_request.number }}" --body-file $path + $lines | Set-Content -Encoding utf8 $env:TEMP/portapy-full-core-windows.md + gh pr comment "${{ github.event.pull_request.number }}" ` + --body-file $env:TEMP/portapy-full-core-windows.md - uses: actions/upload-artifact@v4 if: always() with: diff --git a/.github/workflows/native-functions.yml b/.github/workflows/native-functions.yml deleted file mode 100644 index 36e3d21e..00000000 --- a/.github/workflows/native-functions.yml +++ /dev/null @@ -1,119 +0,0 @@ -name: Native function probe - -on: - pull_request: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - -jobs: - linux: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install native toolchain and asmpython - run: | - sudo apt-get update - sudo apt-get install --yes nasm gcc binutils - python -m pip install --no-cache-dir --force-reinstall \ - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - - name: Compile generated native function entry - run: | - python tools/build_native_functions.py \ - --target linux \ - --output dist/libportapy-functions.so \ - --work-dir dist/build-functions-linux - - name: Execute function entry from C - run: | - cc -std=c11 -Wall -Wextra -Werror -Iinclude \ - tests/native_function_host.c -ldl -o dist/native_function_host - ./dist/native_function_host ./dist/libportapy-functions.so \ - | tee dist/native-function-output.txt - grep -qx 'native-functions: ok' dist/native-function-output.txt - - name: Execute positional variadics from C - run: | - cc -std=c11 -Wall -Wextra -Werror -Iinclude \ - tests/native_varargs_host.c -ldl -o dist/native_varargs_host - ./dist/native_varargs_host ./dist/libportapy-functions.so \ - | tee dist/native-varargs-output.txt - grep -qx 'native-varargs: ok' dist/native-varargs-output.txt - - name: Execute keyword variadics from C - run: | - cc -std=c11 -Wall -Wextra -Werror -Iinclude \ - tests/native_kwargs_host.c -ldl -o dist/native_kwargs_host - ./dist/native_kwargs_host ./dist/libportapy-functions.so \ - | tee dist/native-kwargs-output.txt - grep -qx 'native-kwargs: ok' dist/native-kwargs-output.txt - - uses: actions/upload-artifact@v4 - if: always() - with: - name: linux-native-functions - path: dist - - windows: - runs-on: windows-2025 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install native toolchain and asmpython - shell: powershell - run: | - choco install nasm mingw -y --no-progress - python -m pip install --no-cache-dir --force-reinstall ` - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - - name: Compile generated native function entry - shell: powershell - run: | - python tools/build_native_functions.py ` - --target windows ` - --output dist/portapy-functions.dll ` - --work-dir dist/build-functions-windows - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - name: Execute function entry from C - shell: powershell - run: | - gcc -std=c11 -Wall -Wextra -Werror -Iinclude ` - tests/native_function_host.c -o dist/native_function_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $output = & dist/native_function_host.exe dist/portapy-functions.dll - $output | Out-File -Encoding utf8 dist/native-function-output.txt - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - if ($output -ne 'native-functions: ok') { - throw "unexpected native function output: $output" - } - - name: Execute positional variadics from C - shell: powershell - run: | - gcc -std=c11 -Wall -Wextra -Werror -Iinclude ` - tests/native_varargs_host.c -o dist/native_varargs_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $output = & dist/native_varargs_host.exe dist/portapy-functions.dll - $output | Out-File -Encoding utf8 dist/native-varargs-output.txt - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - if ($output -ne 'native-varargs: ok') { - throw "unexpected native varargs output: $output" - } - - name: Execute keyword variadics from C - shell: powershell - run: | - gcc -std=c11 -Wall -Wextra -Werror -Iinclude ` - tests/native_kwargs_host.c -o dist/native_kwargs_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $output = & dist/native_kwargs_host.exe dist/portapy-functions.dll - $output | Out-File -Encoding utf8 dist/native-kwargs-output.txt - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - if ($output -ne 'native-kwargs: ok') { - throw "unexpected native kwargs output: $output" - } - - uses: actions/upload-artifact@v4 - if: always() - with: - name: windows-native-functions - path: dist diff --git a/.github/workflows/native-handles.yml b/.github/workflows/native-handles.yml deleted file mode 100644 index 203765d9..00000000 --- a/.github/workflows/native-handles.yml +++ /dev/null @@ -1,226 +0,0 @@ -name: Native opaque-handle ABI - -on: - pull_request: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - pull-requests: write - -jobs: - linux: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install native toolchain and asmpython - run: | - sudo apt-get update - sudo apt-get install --yes nasm gcc binutils - python -m pip install --no-cache-dir --force-reinstall \ - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - - name: Build genuine Python-authored library - run: | - python tools/build_native_typed.py \ - --target linux \ - --output dist/libportapy.so \ - --work-dir dist/build-linux - - name: Report Linux build failure - if: failure() && github.event_name == 'pull_request' - env: - GH_TOKEN: ${{ github.token }} - run: | - { - echo '' - echo '### Linux native build diagnostic' - echo - echo '```text' - find dist -maxdepth 3 -type f -print 2>/dev/null || true - echo - for file in \ - dist/build-linux/linux-asmpython-build.log \ - dist/build-linux/linux-nasm.log \ - dist/build-linux/linux-glue.log \ - dist/build-linux/linux-host-glue.log \ - dist/build-linux/linux-host-call-glue.log \ - dist/build-linux/linux-host-call-nasm.log \ - dist/build-linux/linux-host-call-link.log \ - dist/build-linux/linux-link.log; do - if [ -f "$file" ]; then - echo "===== $file =====" - tail -n 120 "$file" - fi - done - echo '```' - } > /tmp/portapy-linux-diagnostic.md - gh pr comment "${{ github.event.pull_request.number }}" \ - --body-file /tmp/portapy-linux-diagnostic.md - - name: Verify exports and execute external hosts - run: | - if readelf -dW dist/libportapy.so | grep -q TEXTREL; then - echo 'text relocations are forbidden' - exit 1 - fi - nm -D --defined-only dist/libportapy.so | tee dist/linux-exports.txt - awk '{print $3}' dist/linux-exports.txt | grep '^portapy_' | sort > dist/actual-symbols.txt - python - <<'PY' - from pathlib import Path - from tools.native_surface import public_exports - Path('dist/expected-symbols.txt').write_text( - '\n'.join(sorted(public_exports(host_bridge=True, host_calls=True))) + '\n', - encoding='utf-8', - ) - PY - diff -u dist/expected-symbols.txt dist/actual-symbols.txt - cc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_handle_host.c -ldl -o dist/native_handle_host - ./dist/native_handle_host ./dist/libportapy.so | tee dist/host-output.txt - grep -qx 'opaque-floats: ok' dist/host-output.txt - cc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_statement_host.c -ldl -o dist/native_statement_host - ./dist/native_statement_host ./dist/libportapy.so | tee dist/statement-output.txt - grep -qx 'statement-blocks: ok' dist/statement-output.txt - cc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_text_error_host.c -ldl -o dist/native_text_error_host - ./dist/native_text_error_host ./dist/libportapy.so | tee dist/text-error-output.txt - grep -qx 'native-text-errors: ok' dist/text-error-output.txt - cc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_typed_literal_host.c -ldl -o dist/native_typed_literal_host - ./dist/native_typed_literal_host ./dist/libportapy.so | tee dist/typed-literal-output.txt - grep -qx 'typed-literals: ok' dist/typed-literal-output.txt - cc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_boolean_expression_host.c -ldl -o dist/native_boolean_expression_host - ./dist/native_boolean_expression_host ./dist/libportapy.so | tee dist/boolean-expression-output.txt - grep -qx 'boolean-expressions: ok' dist/boolean-expression-output.txt - cc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_expression_host.c -ldl -o dist/native_expression_host - ./dist/native_expression_host ./dist/libportapy.so | tee dist/expression-output.txt - grep -qx 'general-expressions: ok' dist/expression-output.txt - cc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_control_flow_host.c -ldl -o dist/native_control_flow_host - ./dist/native_control_flow_host ./dist/libportapy.so | tee dist/control-flow-output.txt - grep -qx 'control-flow: ok' dist/control-flow-output.txt - cc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_function_host.c -ldl -o dist/native_function_host - ./dist/native_function_host ./dist/libportapy.so | tee dist/function-output.txt - grep -qx 'native-functions: ok' dist/function-output.txt - cc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_host_object_host.c -ldl -o dist/native_host_object_host - ./dist/native_host_object_host ./dist/libportapy.so | tee dist/host-object-output.txt - grep -qx 'native-host-objects: ok' dist/host-object-output.txt - cc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_host_call_host.c -ldl -o dist/native_host_call_host - ./dist/native_host_call_host ./dist/libportapy.so | tee dist/host-call-output.txt - grep -qx 'native-host-calls: ok' dist/host-call-output.txt - - uses: actions/upload-artifact@v4 - if: always() - with: - name: linux-native - path: dist - - windows: - runs-on: windows-2025 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install native toolchain and asmpython - shell: powershell - run: | - choco install nasm mingw -y --no-progress - python -m pip install --no-cache-dir --force-reinstall ` - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - - name: Build genuine Python-authored library - shell: powershell - run: | - python tools/build_native_typed.py ` - --target windows ` - --output dist/portapy.dll ` - --work-dir dist/build-windows - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - name: Report Windows build failure - if: failure() && github.event_name == 'pull_request' - shell: powershell - env: - GH_TOKEN: ${{ github.token }} - run: | - $lines = @( - '', - '### Windows native build diagnostic', - '', - '```text' - ) - Get-ChildItem -Path dist -File -Recurse -ErrorAction SilentlyContinue | - ForEach-Object { $lines += $_.FullName } - foreach ($file in @( - 'dist/build-windows/windows-asmpython-build.log', - 'dist/build-windows/windows-nasm.log', - 'dist/build-windows/windows-glue.log', - 'dist/build-windows/windows-host-glue.log', - 'dist/build-windows/windows-host-call-glue.log', - 'dist/build-windows/windows-host-call-nasm.log', - 'dist/build-windows/windows-host-call-link.log', - 'dist/build-windows/windows-link.log' - )) { - if (Test-Path $file) { - $lines += "===== $file =====" - $lines += Get-Content $file -Tail 120 - } - } - $lines += '```' - $lines | Set-Content -Encoding utf8 $env:RUNNER_TEMP\portapy-windows-diagnostic.md - gh pr comment "${{ github.event.pull_request.number }}" ` - --body-file $env:RUNNER_TEMP\portapy-windows-diagnostic.md - - name: Verify exports and execute external hosts - shell: powershell - run: | - objdump -p dist/portapy.dll | Out-File -Encoding utf8 dist/windows-exports.txt - $symbols = python -c "from tools.native_surface import public_exports; print(' '.join(public_exports(host_bridge=True, host_calls=True)))" - foreach ($symbol in $symbols.Split(' ', [System.StringSplitOptions]::RemoveEmptyEntries)) { - if (-not (Select-String -Path dist/windows-exports.txt -Pattern "\b$symbol\b" -Quiet)) { - throw "missing export: $symbol" - } - } - if (Select-String -Path dist/windows-exports.txt -Pattern 'portapy_internal_' -Quiet) { - throw 'internal Python symbols must not be exported' - } - gcc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_handle_host.c -o dist/native_handle_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $output = & dist/native_handle_host.exe dist/portapy.dll - if ($output -ne 'opaque-floats: ok') { throw "unexpected host output: $output" } - gcc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_statement_host.c -o dist/native_statement_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $statementOutput = & dist/native_statement_host.exe dist/portapy.dll - if ($statementOutput -ne 'statement-blocks: ok') { throw "unexpected statement output: $statementOutput" } - gcc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_text_error_host.c -o dist/native_text_error_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $textErrorOutput = & dist/native_text_error_host.exe dist/portapy.dll - if ($textErrorOutput -ne 'native-text-errors: ok') { throw "unexpected text/error output: $textErrorOutput" } - gcc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_typed_literal_host.c -o dist/native_typed_literal_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $typedLiteralOutput = & dist/native_typed_literal_host.exe dist/portapy.dll - if ($typedLiteralOutput -ne 'typed-literals: ok') { throw "unexpected typed literal output: $typedLiteralOutput" } - gcc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_boolean_expression_host.c -o dist/native_boolean_expression_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $booleanExpressionOutput = & dist/native_boolean_expression_host.exe dist/portapy.dll - if ($booleanExpressionOutput -ne 'boolean-expressions: ok') { throw "unexpected boolean expression output: $booleanExpressionOutput" } - gcc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_expression_host.c -o dist/native_expression_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $expressionOutput = & dist/native_expression_host.exe dist/portapy.dll - if ($expressionOutput -ne 'general-expressions: ok') { throw "unexpected expression output: $expressionOutput" } - gcc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_control_flow_host.c -o dist/native_control_flow_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $controlFlowOutput = & dist/native_control_flow_host.exe dist/portapy.dll - if ($controlFlowOutput -ne 'control-flow: ok') { throw "unexpected control-flow output: $controlFlowOutput" } - gcc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_function_host.c -o dist/native_function_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $functionOutput = & dist/native_function_host.exe dist/portapy.dll - if ($functionOutput -ne 'native-functions: ok') { throw "unexpected function output: $functionOutput" } - gcc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_host_object_host.c -o dist/native_host_object_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $hostObjectOutput = & dist/native_host_object_host.exe dist/portapy.dll - if ($hostObjectOutput -ne 'native-host-objects: ok') { throw "unexpected host-object output: $hostObjectOutput" } - gcc -std=c11 -Wall -Wextra -Werror -Iinclude tests/native_host_call_host.c -o dist/native_host_call_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $hostCallOutput = & dist/native_host_call_host.exe dist/portapy.dll - if ($hostCallOutput -ne 'native-host-calls: ok') { throw "unexpected host-call output: $hostCallOutput" } - - uses: actions/upload-artifact@v4 - if: always() - with: - name: windows-native - path: dist diff --git a/.github/workflows/native-host-calls.yml b/.github/workflows/native-host-calls.yml deleted file mode 100644 index 6491ecc5..00000000 --- a/.github/workflows/native-host-calls.yml +++ /dev/null @@ -1,107 +0,0 @@ -name: Native host call probe - -on: - pull_request: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - -jobs: - linux: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install native toolchain and asmpython - run: | - sudo apt-get update - sudo apt-get install --yes nasm gcc binutils - python -m pip install --no-cache-dir --force-reinstall \ - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - - name: Build host-call library - run: | - python tools/build_native_host_calls.py \ - --target linux \ - --output dist/libportapy.so \ - --work-dir dist/build-linux - - name: Execute host calls from C - run: | - cc -std=c11 -Wall -Wextra -Werror -Iinclude \ - tests/native_host_call_host.c -ldl -o dist/native_host_call_host - ./dist/native_host_call_host ./dist/libportapy.so | tee dist/host-call-output.txt - grep -qx 'native-host-calls: ok' dist/host-call-output.txt - - name: Execute public tuple ABI from C - run: | - cc -std=c11 -Wall -Wextra -Werror -Iinclude \ - tests/native_tuple_host.c -ldl -o dist/native_tuple_host - ./dist/native_tuple_host ./dist/libportapy.so | tee dist/tuple-output.txt - grep -qx 'native-tuples: ok' dist/tuple-output.txt - - name: Execute public dictionary ABI from C - run: | - cc -std=c11 -Wall -Wextra -Werror -Iinclude \ - tests/native_dict_host.c -ldl -o dist/native_dict_host - ./dist/native_dict_host ./dist/libportapy.so | tee dist/dict-output.txt - grep -qx 'native-dicts: ok' dist/dict-output.txt - - uses: actions/upload-artifact@v4 - if: always() - with: - name: linux-native-host-calls - path: dist - - windows: - runs-on: windows-2025 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install native toolchain and asmpython - shell: powershell - run: | - choco install nasm mingw -y --no-progress - python -m pip install --no-cache-dir --force-reinstall ` - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - - name: Build host-call library - shell: powershell - run: | - python tools/build_native_host_calls.py ` - --target windows ` - --output dist/portapy.dll ` - --work-dir dist/build-windows - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - name: Execute host calls from C - shell: powershell - run: | - gcc -std=c11 -Wall -Wextra -Werror -Iinclude ` - tests/native_host_call_host.c -o dist/native_host_call_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $output = & dist/native_host_call_host.exe dist/portapy.dll - $output | Out-File -Encoding utf8 dist/host-call-output.txt - if ($output -ne 'native-host-calls: ok') { throw "unexpected host-call output: $output" } - - name: Execute public tuple ABI from C - shell: powershell - run: | - gcc -std=c11 -Wall -Wextra -Werror -Iinclude ` - tests/native_tuple_host.c -o dist/native_tuple_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $output = & dist/native_tuple_host.exe dist/portapy.dll - $output | Out-File -Encoding utf8 dist/tuple-output.txt - if ($output -ne 'native-tuples: ok') { throw "unexpected tuple output: $output" } - - name: Execute public dictionary ABI from C - shell: powershell - run: | - gcc -std=c11 -Wall -Wextra -Werror -Iinclude ` - tests/native_dict_host.c -o dist/native_dict_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $output = & dist/native_dict_host.exe dist/portapy.dll - $output | Out-File -Encoding utf8 dist/dict-output.txt - if ($output -ne 'native-dicts: ok') { throw "unexpected dictionary output: $output" } - - uses: actions/upload-artifact@v4 - if: always() - with: - name: windows-native-host-calls - path: dist diff --git a/.github/workflows/native-host-objects.yml b/.github/workflows/native-host-objects.yml deleted file mode 100644 index 4455efe6..00000000 --- a/.github/workflows/native-host-objects.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Native host object probe - -on: - pull_request: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - -jobs: - linux: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install native toolchain and asmpython - run: | - sudo apt-get update - sudo apt-get install --yes nasm gcc binutils - python -m pip install --no-cache-dir --force-reinstall \ - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - - name: Build host-object library - run: | - python tools/build_native_host.py \ - --target linux \ - --output dist/libportapy-host.so \ - --work-dir dist/build-host-linux - - name: Execute host-object bridge from C - run: | - cc -std=c11 -Wall -Wextra -Werror -Iinclude \ - tests/native_host_object_host.c -ldl -o dist/native_host_object_host - ./dist/native_host_object_host ./dist/libportapy-host.so \ - | tee dist/native-host-output.txt - grep -qx 'native-host-objects: ok' dist/native-host-output.txt - - uses: actions/upload-artifact@v4 - if: always() - with: - name: linux-native-host-objects - path: dist - - windows: - runs-on: windows-2025 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install native toolchain and asmpython - shell: powershell - run: | - choco install nasm mingw -y --no-progress - python -m pip install --no-cache-dir --force-reinstall ` - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - - name: Build host-object library - shell: powershell - run: | - python tools/build_native_host.py ` - --target windows ` - --output dist/portapy-host.dll ` - --work-dir dist/build-host-windows - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - name: Execute host-object bridge from C - shell: powershell - run: | - gcc -std=c11 -Wall -Wextra -Werror -Iinclude ` - tests/native_host_object_host.c -o dist/native_host_object_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $output = & dist/native_host_object_host.exe dist/portapy-host.dll - $output | Out-File -Encoding utf8 dist/native-host-output.txt - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - if ($output -ne 'native-host-objects: ok') { - throw "unexpected native host output: $output" - } - - uses: actions/upload-artifact@v4 - if: always() - with: - name: windows-native-host-objects - path: dist diff --git a/.github/workflows/native-lists.yml b/.github/workflows/native-lists.yml deleted file mode 100644 index a742f8c8..00000000 --- a/.github/workflows/native-lists.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: Native list ABI - -on: - pull_request: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - -jobs: - linux: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install native toolchain and asmpython - run: | - sudo apt-get update - sudo apt-get install --yes nasm gcc binutils - python -m pip install --no-cache-dir --force-reinstall \ - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - python -m pip install -e . - - name: Build native library with list ABI - run: | - python tools/build_native_host_calls.py \ - --target linux \ - --output dist/libportapy.so \ - --work-dir dist/build-lists-linux - - name: Execute public list ABI from C - run: | - cc -std=c11 -Wall -Wextra -Werror -Iinclude \ - tests/native_list_host.c -ldl -o dist/native_list_host - ./dist/native_list_host ./dist/libportapy.so \ - | tee dist/list-output.txt - grep -qx 'native-lists: ok' dist/list-output.txt - - name: Exercise Python list facade - run: | - python tests/native_list_adapter_probe.py ./dist/libportapy.so \ - | tee dist/list-adapter-output.txt - grep -qx 'native-list-adapter: ok' dist/list-adapter-output.txt - - uses: actions/upload-artifact@v4 - if: always() - with: - name: linux-native-lists - path: dist - - windows: - runs-on: windows-2025 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install native toolchain and asmpython - shell: powershell - run: | - choco install nasm mingw -y --no-progress - python -m pip install --no-cache-dir --force-reinstall ` - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - python -m pip install -e . - - name: Build native library with list ABI - shell: powershell - run: | - python tools/build_native_host_calls.py ` - --target windows ` - --output dist/portapy.dll ` - --work-dir dist/build-lists-windows - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - name: Execute public list ABI from C - shell: powershell - run: | - gcc -std=c11 -Wall -Wextra -Werror -Iinclude ` - tests/native_list_host.c -o dist/native_list_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $output = & dist/native_list_host.exe dist/portapy.dll - $output | Out-File -Encoding utf8 dist/list-output.txt - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - if ($output -ne 'native-lists: ok') { - throw "unexpected native list output: $output" - } - - name: Exercise Python list facade - shell: powershell - run: | - $output = @(python tests/native_list_adapter_probe.py dist/portapy.dll) - $output | Out-File -Encoding utf8 dist/list-adapter-output.txt - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $lastLine = $output[-1] - if ($lastLine -ne 'native-list-adapter: ok') { - throw "unexpected native list adapter output: $($output -join [Environment]::NewLine)" - } - - uses: actions/upload-artifact@v4 - if: always() - with: - name: windows-native-lists - path: dist diff --git a/.github/workflows/native-probe.yml b/.github/workflows/native-probe.yml deleted file mode 100644 index bad94549..00000000 --- a/.github/workflows/native-probe.yml +++ /dev/null @@ -1,149 +0,0 @@ -name: Native shared-library probe - -on: - pull_request: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - -jobs: - linux: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install native toolchain and asmpython - run: | - sudo apt-get update - sudo apt-get install --yes nasm gcc binutils - python -m pip install --no-cache-dir --force-reinstall \ - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - - name: Compile Python source and link libportapy_probe.so - shell: bash - run: | - mkdir -p dist - set +e - set -o pipefail - python -m asmpython build src/portapy/native_probe.py \ - --target linux --type library --backend legacy \ - --no-pyinbin-fallback --keep-assembly \ - -o dist/libportapy_probe.so \ - 2>&1 | tee dist/linux-build.log - compiler_status=${PIPESTATUS[0]} - set -e - - if [ ! -f dist/libportapy_probe.asm ]; then - exit "$compiler_status" - fi - - python tools/nasm_module_init.py dist/libportapy_probe.asm \ - --target linux --public-symbol portapy_library_initialize - python tools/nasm_exports.py dist/libportapy_probe.asm \ - --export portapy_library_initialize \ - --export portapy_abi_version \ - --export portapy_opcode_probe - python tools/elf_pic.py dist/libportapy_probe.asm - nasm -f elf64 -w-label-redef-late \ - dist/libportapy_probe.asm -o dist/libportapy_probe.o - cat > dist/portapy_probe.map <<'MAP' - { - global: - portapy_library_initialize; - portapy_abi_version; - portapy_opcode_probe; - local: *; - }; - MAP - gcc -shared dist/libportapy_probe.o -o dist/libportapy_probe.so \ - -Wl,--version-script=dist/portapy_probe.map - - name: Inspect and execute through an external C host - run: | - file dist/libportapy_probe.so - readelf -dW dist/libportapy_probe.so | tee dist/linux-dynamic.txt - if readelf -dW dist/libportapy_probe.so | grep -q TEXTREL; then - echo 'text relocations are forbidden' - exit 1 - fi - nm -D --defined-only dist/libportapy_probe.so | tee dist/linux-exports.txt - awk '{print $3}' dist/linux-exports.txt | grep '^portapy_' | sort > dist/linux-portapy-symbols.txt - printf 'portapy_abi_version\nportapy_library_initialize\nportapy_opcode_probe\n' > dist/expected-symbols.txt - diff -u dist/expected-symbols.txt dist/linux-portapy-symbols.txt - if awk '{print $3}' dist/linux-exports.txt | grep -qx main; then - echo 'main must not be public' - exit 1 - fi - cc -std=c11 -Wall -Wextra -Werror tests/native_probe_host.c -ldl -o dist/native_probe_host - ./dist/native_probe_host ./dist/libportapy_probe.so | tee dist/linux-host-output.txt - grep -qx 'abi=1 opcode=10' dist/linux-host-output.txt - - uses: actions/upload-artifact@v4 - if: always() - with: - name: linux-native-probe - path: dist - - windows: - runs-on: windows-2025 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install native toolchain and asmpython - shell: powershell - run: | - choco install nasm mingw -y --no-progress - python -m pip install --no-cache-dir --force-reinstall ` - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - - name: Compile Python source and link portapy_probe.dll - shell: powershell - run: | - New-Item -ItemType Directory -Force dist | Out-Null - python -m asmpython build src/portapy/native_probe.py ` - --target windows --type library --backend legacy ` - --no-pyinbin-fallback --keep-assembly ` - -o dist/portapy_probe.dll ` - *>&1 | Tee-Object -FilePath dist/windows-build.log - $compilerExit = $LASTEXITCODE - - if (-not (Test-Path dist/portapy_probe.asm)) { - exit $compilerExit - } - python tools/nasm_module_init.py dist/portapy_probe.asm ` - --target windows --public-symbol portapy_library_initialize - python tools/nasm_exports.py dist/portapy_probe.asm ` - --export portapy_library_initialize ` - --export portapy_abi_version ` - --export portapy_opcode_probe - nasm -f win64 -w-label-redef-late ` - dist/portapy_probe.asm -o dist/portapy_probe.obj - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - @' - LIBRARY portapy_probe - EXPORTS - portapy_library_initialize - portapy_abi_version - portapy_opcode_probe - '@ | Set-Content -Encoding ascii dist/portapy_probe.def - gcc -shared dist/portapy_probe.obj dist/portapy_probe.def ` - -o dist/portapy_probe.dll - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - name: Inspect and execute through an external C host - shell: powershell - run: | - objdump -p dist/portapy_probe.dll | Out-File -Encoding utf8 dist/windows-exports.txt - if (-not (Select-String -Path dist/windows-exports.txt -Pattern 'portapy_library_initialize' -Quiet)) { throw 'missing portapy_library_initialize' } - if (-not (Select-String -Path dist/windows-exports.txt -Pattern 'portapy_abi_version' -Quiet)) { throw 'missing portapy_abi_version' } - if (-not (Select-String -Path dist/windows-exports.txt -Pattern 'portapy_opcode_probe' -Quiet)) { throw 'missing portapy_opcode_probe' } - gcc -std=c11 -Wall -Wextra -Werror tests/native_probe_host.c -o dist/native_probe_host.exe - $output = & dist/native_probe_host.exe dist/portapy_probe.dll - $output | Out-File -Encoding utf8 dist/windows-host-output.txt - if ($output -ne 'abi=1 opcode=10') { throw "unexpected host output: $output" } - - uses: actions/upload-artifact@v4 - if: always() - with: - name: windows-native-probe - path: dist diff --git a/.github/workflows/release-3.14.0.yml b/.github/workflows/release-3.14.0.yml new file mode 100644 index 00000000..1ee21318 --- /dev/null +++ b/.github/workflows/release-3.14.0.yml @@ -0,0 +1,169 @@ +name: Build and release PortaPy 3.14.0 + +on: + workflow_dispatch: + inputs: + publish: + description: Publish 3.14.0 after every native gate passes + required: true + type: boolean + default: false + push: + branches: + - release/3.14.0 + +concurrency: + group: portapy-3.14.0-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: write + +jobs: + linux: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install package and native toolchain + run: | + sudo apt-get update + sudo apt-get install --yes nasm gcc binutils + python -m pip install -e '.[test]' + python -m pip install --no-cache-dir --force-reinstall \ + 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' + - name: Build full libportapy.so + run: | + mkdir -p dist + set -o pipefail + python tools/build_native_typed.py \ + --target linux \ + --output dist/libportapy.so \ + --work-dir dist/build-linux \ + 2>&1 | tee dist/build-linux.log + - name: Validate Linux stable artifact + run: | + readelf -dW dist/libportapy.so | tee dist/linux-dynamic-section.txt + ldd dist/libportapy.so | tee dist/linux-dependencies.txt + if grep -Eiq 'NEEDED.*(lib)?python|(^|[[:space:]/])libpython[0-9]' \ + dist/linux-dynamic-section.txt dist/linux-dependencies.txt; then + echo 'standalone library must not load or link libpython' + exit 1 + fi + if readelf -dW dist/libportapy.so | grep -q TEXTREL; then + echo 'text relocations are forbidden' + exit 1 + fi + python tools/run_native_release_conformance.py \ + dist/libportapy.so \ + --output-dir dist/conformance-linux + - uses: actions/upload-artifact@v4 + if: always() + with: + name: portapy-linux-3.14.0 + path: | + dist/build-linux.log + dist/libportapy.so + dist/libportapy.so.json + dist/linux-dynamic-section.txt + dist/linux-dependencies.txt + dist/conformance-linux + + windows: + runs-on: windows-2022 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install package and native toolchain + shell: powershell + run: | + & .\tools\install_windows_toolchain.ps1 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + python -m pip install -e '.[test]' + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & .\tools\install_pinned_asmpython.ps1 + - name: Build full portapy.dll + shell: powershell + run: | + New-Item -ItemType Directory -Force dist | Out-Null + $output = & python tools/build_native_typed.py ` + --target windows ` + --output dist/portapy.dll ` + --work-dir dist/build-windows ` + 2>&1 + $buildStatus = $LASTEXITCODE + $output | Tee-Object -FilePath dist/build-windows.log + if ($buildStatus -ne 0) { exit $buildStatus } + - name: Validate Windows stable artifact + shell: powershell + run: | + $imports = & objdump -p dist/portapy.dll + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $imports | Out-File -Encoding utf8 dist/windows-imports.txt + if ($imports -match 'DLL Name:\s*(python|libpython)[^\s]*\.dll') { + throw 'standalone library must not load or link a Python DLL' + } + python tools/run_native_release_conformance.py ` + dist/portapy.dll ` + --compiler gcc ` + --output-dir dist/conformance-windows + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + - uses: actions/upload-artifact@v4 + if: always() + with: + name: portapy-windows-3.14.0 + path: | + dist/build-windows.log + dist/portapy.dll + dist/portapy.dll.json + dist/windows-imports.txt + dist/conformance-windows + + publish: + if: ${{ github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish) }} + needs: [linux, windows] + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + path: dist-download + - name: Assemble release directory + run: | + mkdir -p dist + cp dist-download/portapy-linux-3.14.0/libportapy.so dist/ + cp dist-download/portapy-linux-3.14.0/libportapy.so.json dist/ + cp dist-download/portapy-windows-3.14.0/portapy.dll dist/ + cp dist-download/portapy-windows-3.14.0/portapy.dll.json dist/ + - name: Validate metadata and checksums + run: | + PYTHONPATH="$PWD" python tools/release_gate.py dist \ + --expected-tag 3.14.0 + cp include/portapy.h dist/portapy.h + cp LICENSE dist/LICENSE.txt + cp RELEASE_STATUS.json dist/RELEASE_STATUS.json + - name: Publish PortaPy 3.14.0 + env: + GH_TOKEN: ${{ github.token }} + run: | + if gh release view 3.14.0 >/dev/null 2>&1; then + echo 'GitHub release 3.14.0 already exists; refusing to replace it.' + exit 1 + fi + gh release create 3.14.0 \ + dist/libportapy.so \ + dist/libportapy.so.json \ + dist/portapy.dll \ + dist/portapy.dll.json \ + dist/portapy.h \ + dist/LICENSE.txt \ + dist/RELEASE_STATUS.json \ + dist/checksums.json \ + dist/release-manifest.json \ + --target "$GITHUB_SHA" \ + --title 'PortaPy 3.14.0' \ + --notes-file dist/RELEASE_NOTES.md diff --git a/.github/workflows/universal-environment-api.yml b/.github/workflows/universal-environment-api.yml deleted file mode 100644 index 9f2dbded..00000000 --- a/.github/workflows/universal-environment-api.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Universal environment API - -on: - pull_request: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - -jobs: - linux: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install toolchain and package - run: | - sudo apt-get update - sudo apt-get install --yes nasm gcc binutils - python -m pip install --no-cache-dir --force-reinstall \ - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - python -m pip install -e . pytest - - name: Test hosted helper API - run: python -m pytest tests/test_environment_api.py - - name: Build shared library - run: | - python tools/build_native_typed.py \ - --target linux \ - --output dist/libportapy.so \ - --work-dir dist/build-universal-linux - - name: Test universal and low-level ABI together - run: | - cc -std=c11 -Wall -Wextra -Werror -Iinclude \ - tests/native_environment_api_host.c -ldl \ - -o dist/native_environment_api_host - ./dist/native_environment_api_host ./dist/libportapy.so \ - | tee dist/universal-environment-output.txt - grep -qx 'Hello, world!' dist/universal-environment-output.txt - tail -n 1 dist/universal-environment-output.txt \ - | grep -qx 'universal-environment-api: ok' - - uses: actions/upload-artifact@v4 - if: always() - with: - name: universal-environment-linux - path: dist - - windows: - runs-on: windows-2025 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install toolchain and package - shell: powershell - run: | - choco install nasm mingw -y --no-progress - python -m pip install --no-cache-dir --force-reinstall ` - 'git+https://github.com/deltathedumb/asmpython.git@376cf9422c28123673a1dedd7dd66b845f3c5ed1' - python -m pip install -e . pytest - - name: Test hosted helper API - shell: powershell - run: python -m pytest tests/test_environment_api.py - - name: Build shared library - shell: powershell - run: | - python tools/build_native_typed.py ` - --target windows ` - --output dist/portapy.dll ` - --work-dir dist/build-universal-windows - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - name: Test universal and low-level ABI together - shell: powershell - run: | - gcc -std=c11 -Wall -Wextra -Werror -Iinclude ` - tests/native_environment_api_host.c ` - -o dist/native_environment_api_host.exe - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $output = @(& dist/native_environment_api_host.exe dist/portapy.dll) - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $output | Out-File -Encoding utf8 dist/universal-environment-output.txt - if ($output[0] -ne 'Hello, world!') { - throw "unexpected callback output: $($output -join [Environment]::NewLine)" - } - if ($output[-1] -ne 'universal-environment-api: ok') { - throw "unexpected final output: $($output -join [Environment]::NewLine)" - } - - uses: actions/upload-artifact@v4 - if: always() - with: - name: universal-environment-windows - path: dist diff --git a/README.md b/README.md index 7fc50e54..45355653 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ PortaPy is a separately versioned, embeddable interpreter project derived from the reusable Python-written core of asmpython's `pyinbin` interpreter. -The interpreter implementation is required to remain Python source compiled by asmpython. The public C ABI and generated assembly passes are only host/build boundaries; they do not implement parsing, evaluation, objects, imports, or exception semantics. +The interpreter implementation remains Python source compiled by asmpython. The public C ABI and generated assembly passes are host/build boundaries; they do not implement parsing, evaluation, objects, imports, or exception semantics. Native artifact names: @@ -12,7 +12,7 @@ Native artifact names: ## Universal public API -The stable cross-language contract is a C ABI. Any language capable of calling C functions can use the same native library. PortaPy does not provide `import_module`; the host language imports or loads its own modules and then adds their objects to an environment. +The stable cross-language contract is a C ABI. Any language capable of calling C functions can use the same native library. PortaPy does not provide `import_module`; the host language loads its own modules and adds their objects to an environment. The first-class helper exports are: @@ -22,59 +22,58 @@ The first-class helper exports are: - `portapy_execute()` and `portapy_evaluate()` - `portapy_destroy()` -`portapy_environment` is an alias of the opaque `portapy_runtime` handle, so a host may freely drop from the helper layer into the complete low-level runtime, value, global, callback, container, and error APIs. +`portapy_environment` aliases the opaque `portapy_runtime` handle, so hosts can freely move between the helper layer and the complete low-level runtime, value, global, callback, container, snapshot, and error APIs. See [`docs/FFI.md`](docs/FFI.md) for C and direct C# P/Invoke examples. ## High-level Python API -The native binary interface is environment-oriented and available through a Python facade: - ```python import math from portapy import import_binary -from somnia import env portapy = import_binary("portapy.dll") environment = portapy.new() -environment.add(math) # available as math.floor(...) -environment.add_all(env) # public members become direct globals -environment.set("requested_value", 41.9) -environment.set("coordinates", (10, 20, (30, 40))) -environment.set("settings", {"scale": 2, "nested": {"value": 21}}) -environment.set("samples", [18, [1, 2], 24]) +environment.add(math) +environment.add_all({"seed": 40}) +environment.set("values", [40, 2]) environment.execute(""" -http_provider = game.provider.HttpProvider -floor_value = math.floor(requested_value) -answer = floor_value + 1 -first_coordinate = coordinates[0] -scaled = settings["nested"]["value"] * settings["scale"] -first_sample = samples[0] -""") +def total(items): + result = 0 + for item in items: + result += item + return result + +class Box: + def __init__(self, value): + self.value = value + + def get(self): + return self.value -snapshot = environment.snapshot() -http_provider = snapshot.var["http_provider"] -answer = snapshot.var["answer"] -coordinates = snapshot.var["coordinates"] -settings = snapshot.var["settings"] -samples = snapshot.var["samples"] +box = Box(value=total(items=values)) +answer = box.get() +""") +assert environment.get("answer") == 42 ``` -`add(value)` binds a named function, class, module, or object using its `__name__`, unless an explicit name is supplied. `add_all(module)` flattens all eligible public members from a module, object, or mapping. `add_module()`, `add_modules()`, `expose()`, and `add_builtin()` remain compatibility or fine-grained namespace operations. +`add(value)` binds a named function, class, module, or object using its `__name__`, unless an explicit name is supplied. `add_all(value)` flattens eligible public members from a module, object, or mapping. `add_module()`, `add_modules()`, `expose()`, and `add_builtin()` remain compatibility or fine-grained namespace operations. -The adapter automatically converts Python `None`, booleans, signed 64-bit integers, floats, strings, bytes, tuples, lists, string-key mappings, modules, objects, and callables into native PortaPy values. Tuples, lists, and mappings are converted recursively and remain ordinary PortaPy values across globals, snapshots, and host callback arguments/results. Native mapping keys are currently restricted to non-empty ASCII strings. Object members become host attribute graphs, while callables are routed through the synchronous callback ABI. +The adapter converts Python `None`, booleans, signed 64-bit integers, floats, strings, bytes, tuples, lists, string-key mappings, modules, objects, and callables into native PortaPy values. Containers are converted recursively and remain ordinary PortaPy values across globals, snapshots, and callback arguments/results. Native mapping keys are restricted to non-empty ASCII strings. -Snapshots capture a shallow, detached set of global bindings. `snapshot.var` is a read-only mapping, while `snapshot.restore()` restores those bindings to the originating environment and deletes globals created after the snapshot. Mutations inside referenced host objects are intentionally not deep-rolled back. +Snapshots capture a shallow detached set of global bindings. `snapshot.var` is read-only, while `snapshot.restore()` restores those bindings and deletes globals created after the snapshot. The hosted implementation uses the same API: ```python import portapy + def plus_one(value): return value + 1 + environment = portapy.new() environment.add(plus_one) environment.add_all({"seed": 41}) @@ -84,91 +83,39 @@ assert environment.snapshot().var["answer"] == 42 ## Fine-grained native ABI -The helper API is built on, and interoperates with, the complete public C ABI: - -- `portapy_runtime_create()` and `portapy_runtime_destroy()` for explicit runtime ownership. -- `portapy_exec_utf8()` and `portapy_eval_utf8()` for explicit UTF-8 spans and filenames. -- `portapy_value_from_host_object()` for opaque objects with stable host IDs. -- `portapy_value_from_host_callable()` for callables with stable callable IDs. -- `portapy_value_from_tuple()` for immutable tuples built from borrowed item handles. -- `portapy_tuple_get_size()` and `portapy_tuple_get_item()` for retained tuple extraction. -- `portapy_value_from_dict()` and `portapy_dict_set_utf8()` for owned string-key dictionaries. -- `portapy_dict_get_size()`, `portapy_dict_key_copy_utf8()`, and `portapy_dict_get_item_utf8()` for enumeration and retained lookup. -- `portapy_value_from_list()` for mutable lists built from borrowed item handles. -- `portapy_list_get_size()`, `portapy_list_get_item()`, `portapy_list_set_item()`, and `portapy_list_append()` for retained extraction and mutation. -- `portapy_set_global_utf8()` and `portapy_delete_global_utf8()` for namespace management. -- `portapy_global_count()` and `portapy_global_name_copy_utf8()` for exact snapshot enumeration. -- `portapy_host_set_attr_utf8()` for host-owned attribute graphs. -- `portapy_host_set_call_handler()` for a raw synchronous dispatcher per runtime. -- helper callbacks and raw callback dispatchers coexisting in one environment. -- checked conversions, retained callback results, structured errors, and retain/release ownership. - -## 3.14 Developer Preview 1 - -`3.14-dev.1` is the first genuine native-library preview. Its runtime state, value ownership, text storage, source parsing, UTF-8 validation, structured error state, control flow, functions, host-object graph, host-call parser, and namespace management are Python-authored and compiled by asmpython. Linux and Windows artifacts are exercised from independent C hosts and from the high-level Python binary facade before publication. - -Implemented native ABI and source surface: - -- isolated runtime and environment handles -- first-class cross-language `new`, `add`, `add_all`, `execute`, `evaluate`, and `destroy` exports -- `None`, normalized `bool`, signed 64-bit integer, bit-exact binary64, string, bytes, tuple, dictionary, list, callable, and opaque object handles -- stable 64-bit host object and callable IDs -- retained native global injection, enumeration, replacement, and deletion -- host attribute graph registration, replacement, lookup, and dotted traversal -- synchronous qualified, flattened, and nested host calls -- helper and fine-grained raw callbacks in the same environment -- borrowed callback arguments, owned callback results, and structured callback failures -- `import_binary()` / `load_native()` Python binary facades -- hosted and native `new()`, `add()`, `add_all()`, `add_modules()`, `expose()`, `set()`, `get()`, `remove()`, `execute()`, `evaluate()`, and snapshots -- automatic Python scalar, tuple, list, string-key mapping, module, object, and callable adaptation -- exact native snapshot restoration with post-snapshot global cleanup -- checked value-kind/conversion and buffer-copy operations -- public tuple construction, size, and retained item extraction -- public dictionary construction, replacement, key enumeration, and retained lookup -- public list construction, size, retained item extraction, replacement, and append -- recursive tuple, dictionary, and list release through normal value ownership -- recursive tuple/list/mapping globals, snapshots, and host callback round-trips -- per-runtime structured error status, type, message, line, and column -- retain/release and runtime-owned teardown -- precedence-aware integer arithmetic, powers, shifts, and bitwise expressions -- string/bytes concatenation and repetition -- native `None`, boolean, quoted string, bytes, tuple, dictionary, and list literals -- empty, single-item, multi-item, and nested tuples -- positive, negative, and chained tuple indexing -- tuple-aware `len()`, truthiness, and recursive structural equality -- owned string-key dictionaries with `len()`, truthiness, equality, and indexing -- mutable lists with positive/negative indexing, `len()`, truthiness, and recursive structural equality -- recursive dictionary and list child ownership -- UTF-8 source literals across hosted Unicode and native byte-oriented source boundaries -- tuple, dictionary, and list values passed through native functions and control flow -- container literals inside native function calls -- equality, ordering, `is`, and `is not` comparisons -- `not`, `and`, and `or` with Python-style truthiness and operand returns -- typed global assignment, lookup, aliasing, augmented assignment, and `eval` -- newline/semicolon statement blocks, bare expressions, and `pass` -- indented `if`/`else`, nested blocks, and `while` -- `break` and `continue` -- positional `def` functions, zero/multi-argument calls, nested calls, and `return` -- recursive `if`/`else` and `while` blocks inside native functions -- nested `break`, `continue`, and early `return` propagation inside functions -- trailing scalar defaults captured once when each `def` executes -- transactional capture replacement on successful function redefinition -- positional/keyword, mixed, reordered, and nested default calls -- `/` positional-only and bare `*` keyword-only parameter markers -- named `*args` parameters packed into real immutable tuple values -- named `**kwargs` parameters packed into owned string-key dictionaries -- mixed fixed, positional-only, keyword-only, positional-variadic, and keyword-variadic binding -- positional-only names captured by `**kwargs`, matching Python behavior -- local call-frame save/restore without leaking variadic bindings -- missing, duplicate, unexpected, parameter-kind, and positional-after-keyword argument errors -- callable value handles and cross-`exec` function persistence -- quote-aware comments and separators -- exact public export allowlists -- Linux position-independent linking with no text relocations -- independent Linux and Windows C and Python conformance hosts +The helper API interoperates with the complete public C ABI: + +- explicit runtime/environment ownership +- UTF-8 source execution and expression evaluation +- scalar, string, bytes, tuple, list, dictionary, callable, and opaque object values +- retained container extraction and mutable list/dictionary operations +- global injection, enumeration, replacement, and deletion +- host attribute graphs and synchronous callback dispatch +- snapshots and exact post-snapshot global cleanup +- checked conversions, buffer copies, retain/release ownership, and structured errors + +See [`include/portapy.h`](include/portapy.h) for the authoritative function surface. + +## PortaPy 3.14.0 + +`3.14.0` is the first source-ready stable release. The canonical Linux and Windows artifacts contain PortaPy's standalone parser, full frontend, bytecode VM, and public embedding ABI. + +The native runtime includes: + +- ordinary source execution and expression evaluation +- functions, defaults, positional-only/keyword-only parameters, `*args`, and `**kwargs` +- nested functions and captured closures +- classes, constructors, instance attributes, and bound methods +- `if`, `while`, `for`, `break`, `continue`, and early return +- tuples, mutable lists, and string-key dictionaries with recursive ownership +- configured import statements inside executed PortaPy source +- exceptions, structured errors, and synthetic traceback frame chains +- host objects, flattened module exposure, and synchronous callbacks +- language-neutral C ABI plus Python and direct C# facades +- Linux and Windows external C/Python conformance suites - reproducible native builds pinned to a verified asmpython compiler commit -This preview is **not** the final standalone Python 3.14 interpreter release. Remaining gates include closures, classes, completing the frontend/bytecode VM transition, broader object syntax, full traceback-frame retrieval, and native module imports inside executed PortaPy source. Host module loading is intentionally outside the PortaPy embedding API. +The release artifacts are `portapy.dll`, `libportapy.so`, `portapy.h`, metadata manifests, FFI examples, and SHA-256 checksums. ## Relationship to pyinbin diff --git a/RELEASE_STATUS.json b/RELEASE_STATUS.json index a1de3d42..69bab868 100644 --- a/RELEASE_STATUS.json +++ b/RELEASE_STATUS.json @@ -1,130 +1,47 @@ { "version_line": "3.14", - "release_tag": "3.14-dev.1", - "stage": "developer-preview", - "prerelease": true, + "release_tag": "3.14.0", + "stage": "stable", + "prerelease": false, "python_built_runtime": true, "native_targets": ["linux-x86_64", "windows-x86_64"], - "source_execution_ready": false, + "source_execution_ready": true, "completed_surface": [ - "runtime handles", - "environment handles aliasing runtime handles", - "first-class native new and new_with_config helpers", - "first-class native add and add_all binding helpers", - "first-class native add_value_utf8 and add_callable_utf8 helpers", - "first-class native execute evaluate and destroy helpers", - "helper and low-level callback dispatch coexistence", - "language-neutral C ABI usable from foreign-function interfaces", - "host-owned module loading without a PortaPy import_module API", - "None values", - "bool values", - "signed 64-bit integer values", - "bit-exact binary64 values", - "UTF-8 string value handles", - "arbitrary bytes value handles", - "opaque host object value handles with stable 64-bit IDs", - "host callable value handles with stable 64-bit IDs", - "checked text and byte buffer copies", - "retain and release", - "runtime isolation", - "precedence-aware integer arithmetic and power expressions", - "integer shifts and bitwise expressions", - "string and bytes concatenation and repetition", - "typed literal evaluation for None bool str and bytes", - "UTF-8 source literal encoding across hosted and native byte-oriented boundaries", - "immutable tuple literals", - "empty single-item multi-item and nested tuples", - "positive negative and chained tuple indexing", - "tuple-aware len truthiness and structural equality", - "tuple values in native functions and control flow", - "tuple element ownership in flat runtime handle storage", - "public immutable tuple construction", - "public tuple size and retained item extraction", - "recursive tuple release through ordinary value ownership", - "recursive Python tuple boxing and unboxing", - "tuple globals snapshots and host callback round-trips", - "owned string-key dictionary values", - "dictionary len truthiness structural equality and string-key indexing", - "recursive dictionary child ownership", - "keyword variadic parameters packed into dictionaries", - "mixed fixed positional-only keyword-only positional variadic and keyword variadic calls", - "positional-only names captured by keyword variadics", - "nested keyword variadic calls and local restoration", - "public dictionary construction replacement size key enumeration and retained lookup", - "public dictionary recursive ownership", - "recursive Python mapping boxing and unboxing", - "dictionary globals snapshots and host callback round-trips", - "owned mutable list values", - "empty nested and trailing-comma list literals", - "positive and negative list indexing", - "list len truthiness and recursive structural equality", - "recursive list child ownership", - "list values in native functions and control flow", - "container literals inside native function calls", - "public list construction size retained extraction replacement and append", - "recursive Python list boxing and unboxing", - "list globals snapshots and host callback round-trips", - "equality and ordering comparisons", - "identity comparisons with is and is not", - "not and or with operand-return semantics", - "truthiness for native scalar handles", - "typed global assignment lookup aliasing and augmented assignment", - "retained native host global injection", - "native host attribute graph registration and lookup", - "dotted host attribute traversal in eval and source execution", - "host ID recovery from evaluated and snapshotted value handles", - "synchronous native host callable dispatch", - "borrowed callback argument handles and owned callback results", - "qualified and flattened host calls", - "nested host calls", - "structured host callback failures", - "native global enumeration and deletion", - "native import_binary and load_native Python facades", - "automatic native add add_all add_modules and expose adaptation", - "automatic Python scalar module object callable tuple list and mapping boxing", - "native snapshot.var enumeration and shallow restore", - "native environment remove and exact post-snapshot cleanup", - "statement blocks with newline and semicolon separators", - "bare expression statements and pass", - "indented if and else blocks", - "nested blocks and while loops", - "break and continue", - "positional function definitions and return statements", - "zero-argument and multi-argument function calls", - "nested direct function calls", - "recursive if and else blocks inside native functions", - "while loops inside native functions", - "break and continue propagation inside native functions", - "early return propagation through nested function blocks", - "trailing scalar default function arguments", - "definition-time default expression capture", - "captured default replacement on function redefinition", - "transactional failed function redefinition", - "named function call arguments", - "mixed positional then keyword calls", - "reordered keyword argument binding", - "positional-only parameters with slash markers", - "keyword-only parameters with bare star markers", - "positional variadic parameters packed into immutable tuples", - "empty nested and mixed positional variadic calls", - "missing duplicate unexpected and parameter-kind argument errors", - "callable value handles", - "cross-exec function persistence", - "local function binding save and restore", - "quote-aware statement separators and comments", - "runtime-global typed value lookup", - "per-runtime structured error status type message line and column", - "high-level new Environment and EnvironmentSnapshot API", - "high-level Environment add and add_all API", - "read-only snapshot.var access and shallow snapshot restore", - "module injection and flattened expose API in the hosted runtime", + "standalone native parser", + "complete frontend and bytecode VM execution", + "runtime and environment handles", + "language-neutral new add add_all execute evaluate and destroy C ABI", + "Linux and Windows native shared libraries", + "None bool signed integer binary64 string and bytes values", + "opaque host objects and host callables with stable 64-bit IDs", + "retain release ownership and runtime isolation", + "public immutable tuple ABI and recursive Python tuple adaptation", + "public mutable list ABI and recursive Python list adaptation", + "public string-key dictionary ABI and recursive Python mapping adaptation", + "tuple list dictionary literals indexing length truthiness and structural equality", + "positional keyword positional-only keyword-only varargs and kwargs binding", + "definition-time defaults and transactional function redefinition", + "nested functions and captured closure state", + "classes constructors instance attributes bound methods and method dispatch", + "if else while for break continue and early return", + "integer float string bytes comparison boolean and bitwise expressions", + "global local nonlocal and augmented assignment semantics", + "configured import statements and module registration inside executed source", + "nested exceptions and synthetic traceback frame chains", + "structured status type message line and column errors", + "host attribute graph registration and dotted traversal", + "synchronous qualified flattened and nested host calls", + "borrowed callback arguments and owned callback results", + "native global injection enumeration replacement and deletion", + "snapshots shallow restore and exact post-snapshot cleanup", + "host-owned module loading through add and add_all", + "import_binary and load_native Python facades", + "automatic Python scalar container module object and callable boxing", + "direct C CSharp and Python foreign-function integration", "declared Python binary-module export metadata", - "Linux and Windows external C and Python conformance hosts" + "reproducible builds pinned to a verified asmpython compiler commit", + "external Linux C and Python conformance hosts", + "external Windows C CSharp and Python conformance hosts" ], - "release_blockers": [ - "closures classes and complete frontend bytecode VM execution", - "broader object syntax in the standalone native source parser", - "full traceback frame retrieval beyond structured error location", - "native import statements and module registration inside executed PortaPy source" - ] + "release_blockers": [] } diff --git a/pyproject.toml b/pyproject.toml index 19876880..2edab0e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "portapy" -version = "3.14.0.dev1" +version = "3.14.0" description = "Fully Python-built embeddable interpreter forked from pyinbin" readme = "README.md" requires-python = ">=3.11" diff --git a/src/portapy/__init__.py b/src/portapy/__init__.py index 598d82b0..31015b10 100644 --- a/src/portapy/__init__.py +++ b/src/portapy/__init__.py @@ -17,18 +17,21 @@ import_binary, load_native, ) -# Install recursive container boxing and public environment helpers before any -# native module instance is created. +# Install recursive container boxing, opaque VM-object handling, runtime-private +# global filtering, and public environment helpers before any native module +# instance is created. from . import native_tuple_binary as _native_tuple_binary from . import native_dict_binary as _native_dict_binary from . import native_list_binary as _native_list_binary +from .native_object_binary import NativeObjectReference +from . import native_internal_globals as _native_internal_globals from . import native_environment_helpers as _native_environment_helpers from .reference_api import ErrorInfo, Runtime, Status, ValueKind Snapshot = EnvironmentSnapshot PortaPyExecutionError = ExecutionError -__version__ = "3.14.0-dev" +__version__ = "3.14.0" __all__ = [ "BindingError", @@ -41,6 +44,7 @@ "NativeEnvironment", "NativeEnvironmentSnapshot", "NativeHostReference", + "NativeObjectReference", "NativePortaPyModule", "PortaPyError", "PortaPyExecutionError", diff --git a/src/portapy/native_full_core_probe.py b/src/portapy/native_full_core_probe.py index 174d850e..fc8d0b6f 100644 --- a/src/portapy/native_full_core_probe.py +++ b/src/portapy/native_full_core_probe.py @@ -23,6 +23,7 @@ _portapy_value_get_kind_impl, _runtime, ) +from .reference_api import Status class _ProbeModule: @@ -53,7 +54,7 @@ def portapy_full_core_probe() -> int: instance = _runtime(runtime) if instance is None: return -6 - if instance.set_global("__pyinbin_import__", _probe_import) != PORTAPY_OK: + if instance.set_global("__pyinbin_import__", _probe_import) is not Status.OK: return -7 forty = _portapy_value_from_i64_impl(runtime, 40) two = _portapy_value_from_i64_impl(runtime, 2) @@ -89,6 +90,7 @@ def fail(): """ status = _portapy_exec_span_impl(runtime, source, len(source)) if status != PORTAPY_OK: + print("FULL CORE PROBE ERROR", instance.last_error()) return -1 handle = _portapy_get_global_span_impl(runtime, "answer", 6) if _portapy_value_get_kind_impl(runtime, handle) != PORTAPY_VALUE_INT: diff --git a/src/portapy/native_internal_globals.py b/src/portapy/native_internal_globals.py new file mode 100644 index 00000000..ebd4ec01 --- /dev/null +++ b/src/portapy/native_internal_globals.py @@ -0,0 +1,36 @@ +"""Hide PortaPy runtime-internal globals from the public native facade.""" +from __future__ import annotations + +from . import native_binary as _native + + +_INTERNAL_PREFIXES = ("__pyinbin_", "__portapy_internal_") +_installed = False + + +def _is_public_global(name: str) -> bool: + return not any(name.startswith(prefix) for prefix in _INTERNAL_PREFIXES) + + +def install() -> None: + global _installed + if _installed: + return + _installed = True + + original_global_names = _native.NativeEnvironment._global_names + + def public_global_names(environment: _native.NativeEnvironment) -> tuple[str, ...]: + return tuple( + name + for name in original_global_names(environment) + if _is_public_global(name) + ) + + _native.NativeEnvironment._global_names = public_global_names + + +install() + + +__all__ = ["install"] diff --git a/src/portapy/native_object_binary.py b/src/portapy/native_object_binary.py new file mode 100644 index 00000000..11abb641 --- /dev/null +++ b/src/portapy/native_object_binary.py @@ -0,0 +1,72 @@ +"""Opaque PortaPy-owned object support for the native Python facade.""" +from __future__ import annotations + +import ctypes +from dataclasses import dataclass + +from . import native_binary as _native +from .reference_api import Status, ValueKind + + +@dataclass(frozen=True) +class NativeObjectReference: + """Opaque reference to an object owned by the native PortaPy VM. + + The public ABI exposes host IDs only for host-owned objects. PortaPy-created + classes and instances therefore remain opaque when read through the Python + binary facade rather than being misidentified as host objects. + """ + + +_installed = False + + +def install() -> None: + global _installed + if _installed: + return + _installed = True + + original_unbox = _native.NativeEnvironment._unbox + + def unbox(environment: _native.NativeEnvironment, handle: int) -> object: + kind = _native._STATUS(0) + environment._check( + int( + environment._api.portapy_value_get_kind( + environment._runtime, + _native._U64(handle), + ctypes.byref(kind), + ) + ), + "inspect value kind", + ) + if ValueKind(kind.value) is not ValueKind.OBJECT: + return original_unbox(environment, handle) + + host_id = _native._U64(0) + status = int( + environment._api.portapy_value_get_host_id( + environment._runtime, + _native._U64(handle), + ctypes.byref(host_id), + ) + ) + if status == int(Status.OK): + return environment._objects.get( + int(host_id.value), + _native.NativeHostReference(int(host_id.value)), + ) + if status == int(Status.TYPE_ERROR): + environment._api.portapy_error_clear(environment._runtime) + return NativeObjectReference() + environment._check(status, "recover host object") + raise AssertionError("unreachable native object status") + + _native.NativeEnvironment._unbox = unbox + + +install() + + +__all__ = ["NativeObjectReference", "install"] diff --git a/src/portapy/reference_api.py b/src/portapy/reference_api.py index c9a01120..ef481430 100644 --- a/src/portapy/reference_api.py +++ b/src/portapy/reference_api.py @@ -11,6 +11,7 @@ import traceback from .core.frontend import compile_source +from .core.loader import default_builtins from .core.vm import VirtualMachine @@ -56,8 +57,10 @@ class _Slot: class Runtime: def __init__(self) -> None: self._vm = VirtualMachine() - self._globals: dict[str, object] = {} - self._globals.update({"__name__": "__main__", "__package__": "", "__doc__": None}) + self._globals: dict[str, object] = default_builtins() + self._globals.update( + {"__name__": "__main__", "__package__": "", "__doc__": None} + ) self._values: dict[int, _Slot] = {} self._next = 1 self._eval_counter = 0 diff --git a/tests/native_boolean_expression_host.c b/tests/native_boolean_expression_host.c index 35daf185..fd71fb30 100644 --- a/tests/native_boolean_expression_host.c +++ b/tests/native_boolean_expression_host.c @@ -9,13 +9,69 @@ #define LOAD_LIBRARY(path) ((void *)LoadLibraryA(path)) #define LOAD_SYMBOL(lib, name) ((void *)(uintptr_t)GetProcAddress((HMODULE)(lib), (name))) #define ABI_CALL __cdecl + +static LONG WINAPI portapy_boolean_crash_filter(EXCEPTION_POINTERS *exception) { + DWORD code = 0; + void *address = NULL; + CONTEXT *context = NULL; + if (exception != NULL) { + context = exception->ContextRecord; + if (exception->ExceptionRecord != NULL) { + code = exception->ExceptionRecord->ExceptionCode; + address = exception->ExceptionRecord->ExceptionAddress; + } + } + + HMODULE module = NULL; + char module_name[MAX_PATH] = {0}; + unsigned long long offset = 0; + if (address != NULL && GetModuleHandleExA( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCSTR)address, + &module + )) { + GetModuleFileNameA(module, module_name, (DWORD)sizeof(module_name)); + offset = (unsigned long long)((uintptr_t)address - (uintptr_t)module); + } + + fprintf( + stderr, + "boolean-crash: code=0x%08lx address=%p module=%s offset=0x%llx\n", + (unsigned long)code, + address, + module_name[0] == '\0' ? "" : module_name, + offset + ); +#if defined(_M_X64) || defined(__x86_64__) + if (context != NULL) { + fprintf( + stderr, + "boolean-crash-context: rip=0x%llx rsp=0x%llx rbp=0x%llx\n", + (unsigned long long)context->Rip, + (unsigned long long)context->Rsp, + (unsigned long long)context->Rbp + ); + } +#endif + fflush(stderr); + return EXCEPTION_EXECUTE_HANDLER; +} + +#define INSTALL_CRASH_HANDLER() SetUnhandledExceptionFilter(portapy_boolean_crash_filter) #else #include #define LOAD_LIBRARY(path) dlopen((path), RTLD_NOW | RTLD_LOCAL) #define LOAD_SYMBOL(lib, name) dlsym((lib), (name)) #define ABI_CALL +#define INSTALL_CRASH_HANDLER() ((void)0) #endif +#define TRACE_STEP(message) do { \ + fprintf(stderr, "boolean-step: %s\n", (message)); \ + fflush(stderr); \ +} while (0) + typedef portapy_status (ABI_CALL *initialize_fn)(void); typedef portapy_status (ABI_CALL *runtime_create_fn)(const portapy_config *, portapy_runtime *); typedef portapy_status (ABI_CALL *runtime_destroy_fn)(portapy_runtime); @@ -47,6 +103,21 @@ typedef portapy_status (ABI_CALL *error_get_info_fn)(portapy_runtime, portapy_er return 10; \ } +static portapy_status execute( + exec_utf8_fn function, + portapy_runtime runtime, + const char *source, + const char *filename +) { + return function( + runtime, + (const uint8_t *)source, + strlen(source), + (const uint8_t *)filename, + strlen(filename) + ); +} + static portapy_status evaluate( eval_utf8_fn function, portapy_runtime runtime, @@ -97,9 +168,12 @@ static int expect_data( int main(int argc, char **argv) { if (argc != 2) return 2; + INSTALL_CRASH_HANDLER(); + TRACE_STEP("load-library"); void *library = LOAD_LIBRARY(argv[1]); if (library == NULL) return 3; + TRACE_STEP("resolve-symbols"); RESOLVE(initialize_fn, initialize, "portapy_library_initialize"); RESOLVE(runtime_create_fn, runtime_create, "portapy_runtime_create"); RESOLVE(runtime_destroy_fn, runtime_destroy, "portapy_runtime_destroy"); @@ -114,11 +188,49 @@ int main(int argc, char **argv) { RESOLVE(value_release_fn, value_release, "portapy_value_release"); RESOLVE(error_get_info_fn, error_get_info, "portapy_error_get_info"); + TRACE_STEP("initialize"); if (initialize() != PORTAPY_OK) return 11; portapy_config config = {0}; config.struct_size = sizeof(config); config.abi_version = PORTAPY_ABI_VERSION; + + portapy_runtime preflight = PORTAPY_NULL_RUNTIME; + TRACE_STEP("preflight-runtime-create"); + if (runtime_create(&config, &preflight) != PORTAPY_OK || preflight == 0) return 38; + const char *preflight_sources[] = { + "empty = \"\"\n", + "name = \"Somnia\"\n", + "alias = name\n", + "other = \"PortaPy\"\n", + "zero = 0\n", + "answer = 42\n", + "correct = name == \"Somnia\"\n", + "selected = empty or name\n", + "guarded = name and answer\n" + }; + const char *preflight_steps[] = { + "preflight-empty", + "preflight-name", + "preflight-alias", + "preflight-other", + "preflight-zero", + "preflight-answer", + "preflight-correct", + "preflight-selected", + "preflight-guarded" + }; + const size_t preflight_count = sizeof(preflight_sources) / sizeof(preflight_sources[0]); + for (size_t index = 0; index < preflight_count; ++index) { + TRACE_STEP(preflight_steps[index]); + if (execute(exec_utf8, preflight, preflight_sources[index], "boolean_preflight.py") != PORTAPY_OK) { + return (int)(39 + index); + } + } + TRACE_STEP("preflight-runtime-destroy"); + if (runtime_destroy(preflight) != PORTAPY_OK) return 48; + portapy_runtime runtime = PORTAPY_NULL_RUNTIME; + TRACE_STEP("runtime-create"); if (runtime_create(&config, &runtime) != PORTAPY_OK || runtime == 0) return 12; const char source[] = @@ -131,62 +243,71 @@ int main(int argc, char **argv) { "correct = name == \"Somnia\"\n" "selected = empty or name\n" "guarded = name and answer\n"; - if (exec_utf8( - runtime, - (const uint8_t *)source, - sizeof(source) - 1, - (const uint8_t *)"boolean_block.py", - strlen("boolean_block.py") - ) != PORTAPY_OK) return 13; + TRACE_STEP("exec-boolean-block"); + if (execute(exec_utf8, runtime, source, "boolean_block.py") != PORTAPY_OK) return 13; + TRACE_STEP("eval-arithmetic-equality"); if (!expect_bool(eval_utf8, value_as_bool, value_release, runtime, "40 + 2 == 42", 1)) return 14; + TRACE_STEP("eval-bool-int-equality"); if (!expect_bool(eval_utf8, value_as_bool, value_release, runtime, "True == 1", 1)) return 15; + TRACE_STEP("eval-string-order"); if (!expect_bool(eval_utf8, value_as_bool, value_release, runtime, "\"abc\" < \"abd\"", 1)) return 16; + TRACE_STEP("eval-none-identity"); if (!expect_bool(eval_utf8, value_as_bool, value_release, runtime, "None is None", 1)) return 17; + TRACE_STEP("eval-alias-identity"); if (!expect_bool(eval_utf8, value_as_bool, value_release, runtime, "name is alias", 1)) return 18; + TRACE_STEP("eval-other-nonidentity"); if (!expect_bool(eval_utf8, value_as_bool, value_release, runtime, "name is not other", 1)) return 19; + TRACE_STEP("eval-not-empty"); if (!expect_bool(eval_utf8, value_as_bool, value_release, runtime, "not empty", 1)) return 20; + TRACE_STEP("eval-and-expression"); if (!expect_bool(eval_utf8, value_as_bool, value_release, runtime, "answer > 40 and name == \"Somnia\"", 1)) return 21; portapy_value value = PORTAPY_NULL_VALUE; + TRACE_STEP("eval-or-value"); if (evaluate(eval_utf8, runtime, "empty or name", &value) != PORTAPY_OK) return 22; static const uint8_t somnia[] = {'S', 'o', 'm', 'n', 'i', 'a'}; + TRACE_STEP("read-or-value"); if (!expect_data(value_get_size, value_copy_data, runtime, value, somnia, sizeof(somnia))) return 23; + TRACE_STEP("release-or-value"); if (value_release(runtime, value) != PORTAPY_OK) return 24; + TRACE_STEP("eval-and-value"); if (evaluate(eval_utf8, runtime, "name and answer", &value) != PORTAPY_OK) return 25; int64_t integer = 0; + TRACE_STEP("read-and-value"); if (value_as_i64(runtime, value, &integer) != PORTAPY_OK || integer != 42) return 26; + TRACE_STEP("release-and-value"); if (value_release(runtime, value) != PORTAPY_OK) return 27; const char *global_name = "correct"; - if (get_global( - runtime, - (const uint8_t *)global_name, - strlen(global_name), - &value - ) != PORTAPY_OK) return 28; + TRACE_STEP("get-correct-global"); + if (get_global(runtime, (const uint8_t *)global_name, strlen(global_name), &value) != PORTAPY_OK) return 28; int boolean = 0; + TRACE_STEP("read-correct-global"); if (value_as_bool(runtime, value, &boolean) != PORTAPY_OK || boolean != 1) return 29; + TRACE_STEP("release-correct-global"); if (value_release(runtime, value) != PORTAPY_OK) return 30; global_name = "selected"; - if (get_global( - runtime, - (const uint8_t *)global_name, - strlen(global_name), - &value - ) != PORTAPY_OK) return 31; + TRACE_STEP("get-selected-global"); + if (get_global(runtime, (const uint8_t *)global_name, strlen(global_name), &value) != PORTAPY_OK) return 31; + TRACE_STEP("read-selected-global"); if (!expect_data(value_get_size, value_copy_data, runtime, value, somnia, sizeof(somnia))) return 32; + TRACE_STEP("release-selected-global"); if (value_release(runtime, value) != PORTAPY_OK) return 33; + TRACE_STEP("eval-mixed-type-error"); if (evaluate(eval_utf8, runtime, "\"text\" < 4", &value) != PORTAPY_TYPE_ERROR) return 34; if (value != PORTAPY_NULL_VALUE) return 35; portapy_error_info info = {0}; info.struct_size = sizeof(info); + TRACE_STEP("read-mixed-type-error"); if (error_get_info(runtime, &info) != PORTAPY_OK || info.status != PORTAPY_TYPE_ERROR) return 36; + TRACE_STEP("runtime-destroy"); if (runtime_destroy(runtime) != PORTAPY_OK) return 37; + TRACE_STEP("complete"); puts("boolean-expressions: ok"); return 0; } diff --git a/tests/native_control_flow_host.c b/tests/native_control_flow_host.c index acba2a7e..54b0f0dc 100644 --- a/tests/native_control_flow_host.c +++ b/tests/native_control_flow_host.c @@ -1,7 +1,9 @@ +#define _GNU_SOURCE #include "portapy.h" #include #include +#include #include #if defined(_WIN32) @@ -9,12 +11,118 @@ #define LOAD_LIBRARY(path) ((void *)LoadLibraryA(path)) #define LOAD_SYMBOL(lib, name) ((void *)(uintptr_t)GetProcAddress((HMODULE)(lib), (name))) #define ABI_CALL __cdecl + +static LONG WINAPI portapy_control_crash_filter(EXCEPTION_POINTERS *exception) { + DWORD code = 0; + void *address = NULL; + CONTEXT *context = NULL; + if (exception != NULL) { + context = exception->ContextRecord; + if (exception->ExceptionRecord != NULL) { + code = exception->ExceptionRecord->ExceptionCode; + address = exception->ExceptionRecord->ExceptionAddress; + } + } + HMODULE module = NULL; + char module_name[MAX_PATH] = {0}; + unsigned long long offset = 0; + if (address != NULL && GetModuleHandleExA( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCSTR)address, + &module + )) { + GetModuleFileNameA(module, module_name, (DWORD)sizeof(module_name)); + offset = (unsigned long long)((uintptr_t)address - (uintptr_t)module); + } + fprintf( + stderr, + "control-crash: code=0x%08lx address=%p module=%s offset=0x%llx\n", + (unsigned long)code, + address, + module_name[0] == '\0' ? "" : module_name, + offset + ); +#if defined(_M_X64) || defined(__x86_64__) + if (context != NULL) { + fprintf( + stderr, + "control-crash-context: rip=0x%llx rsp=0x%llx rbp=0x%llx\n", + (unsigned long long)context->Rip, + (unsigned long long)context->Rsp, + (unsigned long long)context->Rbp + ); + } +#endif + fflush(stderr); + return EXCEPTION_EXECUTE_HANDLER; +} + +#define INSTALL_CRASH_HANDLER() SetUnhandledExceptionFilter(portapy_control_crash_filter) #else #include +#include +#include +#include #define LOAD_LIBRARY(path) dlopen((path), RTLD_NOW | RTLD_LOCAL) #define LOAD_SYMBOL(lib, name) dlsym((lib), (name)) #define ABI_CALL + +static void portapy_control_signal_handler( + int signal_number, + siginfo_t *signal_info, + void *context_pointer +) { + uintptr_t instruction = 0; +#if defined(__x86_64__) && defined(REG_RIP) + ucontext_t *context = (ucontext_t *)context_pointer; + instruction = (uintptr_t)context->uc_mcontext.gregs[REG_RIP]; +#else + (void)context_pointer; #endif + Dl_info module_info = {0}; + const char *module_name = ""; + uintptr_t offset = 0; + if (instruction != 0 && dladdr((void *)instruction, &module_info) != 0) { + if (module_info.dli_fname != NULL) module_name = module_info.dli_fname; + if (module_info.dli_fbase != NULL) { + offset = instruction - (uintptr_t)module_info.dli_fbase; + } + } + fprintf( + stderr, + "control-crash: signal=%d fault=%p instruction=%p module=%s offset=0x%llx\n", + signal_number, + signal_info == NULL ? NULL : signal_info->si_addr, + (void *)instruction, + module_name, + (unsigned long long)offset + ); + void *frames[32]; + int frame_count = backtrace(frames, (int)(sizeof(frames) / sizeof(frames[0]))); + backtrace_symbols_fd(frames, frame_count, 2); + fflush(stderr); + _Exit(128 + signal_number); +} + +static void install_control_crash_handler(void) { + struct sigaction action; + memset(&action, 0, sizeof(action)); + action.sa_sigaction = portapy_control_signal_handler; + action.sa_flags = SA_SIGINFO | SA_RESETHAND; + sigemptyset(&action.sa_mask); + sigaction(SIGSEGV, &action, NULL); + sigaction(SIGBUS, &action, NULL); + sigaction(SIGABRT, &action, NULL); +} + +#define INSTALL_CRASH_HANDLER() install_control_crash_handler() +#endif + +#define TRACE_STEP(message) do { \ + fprintf(stderr, "control-step: %s\n", (message)); \ + fflush(stderr); \ +} while (0) typedef portapy_status (ABI_CALL *initialize_fn)(void); typedef portapy_status (ABI_CALL *runtime_create_fn)(const portapy_config *, portapy_runtime *); @@ -66,11 +174,29 @@ static int expect_text( return copied == expected_size && memcmp(buffer, expected, expected_size) == 0; } +static portapy_status execute_text( + exec_utf8_fn execute, + portapy_runtime runtime, + const char *source, + const char *filename +) { + return execute( + runtime, + (const uint8_t *)source, + strlen(source), + (const uint8_t *)filename, + strlen(filename) + ); +} + int main(int argc, char **argv) { if (argc != 2) return 2; + INSTALL_CRASH_HANDLER(); + TRACE_STEP("load-library"); void *library = LOAD_LIBRARY(argv[1]); if (library == NULL) return 3; + TRACE_STEP("resolve-symbols"); RESOLVE(initialize_fn, initialize, "portapy_library_initialize"); RESOLVE(runtime_create_fn, runtime_create, "portapy_runtime_create"); RESOLVE(runtime_destroy_fn, runtime_destroy, "portapy_runtime_destroy"); @@ -83,11 +209,82 @@ int main(int argc, char **argv) { RESOLVE(value_release_fn, value_release, "portapy_value_release"); RESOLVE(error_get_info_fn, error_get_info, "portapy_error_get_info"); + TRACE_STEP("initialize"); if (initialize() != PORTAPY_OK) return 11; portapy_config config = {0}; config.struct_size = sizeof(config); config.abi_version = PORTAPY_ABI_VERSION; + + const char *preflight_sources[] = { + "name = \"Somnia\"\n" + "choice = \"unset\"\n", + "name = \"Somnia\"\n" + "choice = \"unset\"\n" + "if name == \"Somnia\":\n" + " choice = \"matched\"\n" + " if 2 < 3:\n" + " nested = True\n" + "else:\n" + " choice = \"wrong\"\n", + "name = \"Somnia\"\n" + "choice = \"unset\"\n" + "if name == \"Somnia\":\n" + " choice = \"matched\"\n" + " if 2 < 3:\n" + " nested = True\n" + "else:\n" + " choice = \"wrong\"\n" + "count = 0\n" + "total = 0\n" + "while count < 6:\n" + " count = count + 1\n" + " if count == 2:\n" + " continue\n" + " if count == 5:\n" + " break\n" + " total = total + count\n", + "name = \"Somnia\"\n" + "choice = \"unset\"\n" + "if name == \"Somnia\":\n" + " choice = \"matched\"\n" + " if 2 < 3:\n" + " nested = True\n" + "else:\n" + " choice = \"wrong\"\n" + "count = 0\n" + "total = 0\n" + "while count < 6:\n" + " count = count + 1\n" + " if count == 2:\n" + " continue\n" + " if count == 5:\n" + " break\n" + " total = total + count\n" + "finished = count == 5 and total == 8\n" + "42 == 42\n" + "pass\n" + }; + const char *preflight_steps[] = { + "preflight-assignments", + "preflight-if", + "preflight-while", + "preflight-complete" + }; + const size_t preflight_count = sizeof(preflight_sources) / sizeof(preflight_sources[0]); + for (size_t index = 0; index < preflight_count; ++index) { + portapy_runtime probe = PORTAPY_NULL_RUNTIME; + TRACE_STEP(preflight_steps[index]); + if (runtime_create(&config, &probe) != PORTAPY_OK || probe == 0) { + return (int)(40 + index * 2); + } + if (execute_text(exec_utf8, probe, preflight_sources[index], "control_preflight.py") != PORTAPY_OK) { + return (int)(41 + index * 2); + } + if (runtime_destroy(probe) != PORTAPY_OK) return (int)(48 + index); + } + portapy_runtime runtime = PORTAPY_NULL_RUNTIME; + TRACE_STEP("runtime-create"); if (runtime_create(&config, &runtime) != PORTAPY_OK || runtime == 0) return 12; const char source[] = @@ -111,38 +308,49 @@ int main(int argc, char **argv) { "finished = count == 5 and total == 8\n" "42 == 42\n" "pass\n"; - if (exec_utf8( - runtime, - (const uint8_t *)source, - sizeof(source) - 1, - (const uint8_t *)"control_flow.py", - strlen("control_flow.py") - ) != PORTAPY_OK) return 13; + TRACE_STEP("exec-control-block"); + if (execute_text(exec_utf8, runtime, source, "control_flow.py") != PORTAPY_OK) return 13; portapy_value value = PORTAPY_NULL_VALUE; + TRACE_STEP("get-choice"); if (!global_value(get_global, runtime, "choice", &value)) return 14; + TRACE_STEP("read-choice"); if (!expect_text(value_get_size, value_copy_data, runtime, value, "matched")) return 15; + TRACE_STEP("release-choice"); if (value_release(runtime, value) != PORTAPY_OK) return 16; + TRACE_STEP("get-nested"); if (!global_value(get_global, runtime, "nested", &value)) return 17; int boolean = 0; + TRACE_STEP("read-nested"); if (value_as_bool(runtime, value, &boolean) != PORTAPY_OK || boolean != 1) return 18; + TRACE_STEP("release-nested"); if (value_release(runtime, value) != PORTAPY_OK) return 19; + TRACE_STEP("get-count"); if (!global_value(get_global, runtime, "count", &value)) return 20; int64_t integer = 0; + TRACE_STEP("read-count"); if (value_as_i64(runtime, value, &integer) != PORTAPY_OK || integer != 5) return 21; + TRACE_STEP("release-count"); if (value_release(runtime, value) != PORTAPY_OK) return 22; + TRACE_STEP("get-total"); if (!global_value(get_global, runtime, "total", &value)) return 23; + TRACE_STEP("read-total"); if (value_as_i64(runtime, value, &integer) != PORTAPY_OK || integer != 8) return 24; + TRACE_STEP("release-total"); if (value_release(runtime, value) != PORTAPY_OK) return 25; + TRACE_STEP("get-finished"); if (!global_value(get_global, runtime, "finished", &value)) return 26; + TRACE_STEP("read-finished"); if (value_as_bool(runtime, value, &boolean) != PORTAPY_OK || boolean != 1) return 27; + TRACE_STEP("release-finished"); if (value_release(runtime, value) != PORTAPY_OK) return 28; const char invalid[] = "value = 1\n unexpected = 2\n"; + TRACE_STEP("exec-invalid-source"); if (exec_utf8( runtime, (const uint8_t *)invalid, @@ -152,10 +360,14 @@ int main(int argc, char **argv) { ) != PORTAPY_COMPILE_ERROR) return 29; portapy_error_info info = {0}; info.struct_size = sizeof(info); + TRACE_STEP("read-invalid-error"); if (error_get_info(runtime, &info) != PORTAPY_OK) return 30; + TRACE_STEP("validate-invalid-error"); if (info.status != PORTAPY_COMPILE_ERROR || info.line != 2) return 31; + TRACE_STEP("runtime-destroy"); if (runtime_destroy(runtime) != PORTAPY_OK) return 32; + TRACE_STEP("complete"); puts("control-flow: ok"); return 0; } diff --git a/tests/native_environment_adapter_probe.py b/tests/native_environment_adapter_probe.py index 4c8ec267..4e84e86a 100644 --- a/tests/native_environment_adapter_probe.py +++ b/tests/native_environment_adapter_probe.py @@ -52,9 +52,13 @@ def main() -> int: "input_mapping", {"left": 18, "right": 24, "nested": {"value": 42}}, ) + environment.set("values", [40, 2]) environment.execute( + "import math\n" + "from math import floor as imported_floor\n" + "unicode_text = 'π'\n" "http_provider = game.provider.HttpProvider\n" - "floor_value = math.floor(input_value)\n" + "floor_value = imported_floor(input_value)\n" "answer = floor_value + 1\n" "nested = add(20, add(1, 21))\n" "tuple_first = input_tuple[0]\n" @@ -63,15 +67,54 @@ def main() -> int: "mapping_total = input_mapping[\"left\"] + input_mapping[\"right\"]\n" "mapping_size = len(input_mapping)\n" "mapping_result = dict_roundtrip(input_mapping)\n" + "def total(items):\n" + " result = 0\n" + " for item in items:\n" + " result += item\n" + " return result\n" + "def outer(base):\n" + " def inner(value):\n" + " return base + value\n" + " return inner\n" + "class Box:\n" + " def __init__(self, value):\n" + " self.value = value\n" + " def get(self):\n" + " return self.value\n" + "fn = outer(base=19)\n" + "box = Box(value=fn(value=total(items=values) - 19))\n" + "def fail():\n" + " return 1 // 0\n" + "try:\n" + " fail()\n" + "except Exception as exc:\n" + " traced = exc.__traceback__ is not None\n" + "full_runtime_answer = box.get() if traced else -1\n" ) + assert environment.evaluate("not ''") is True + assert environment.evaluate("'alpha' == 'alpha'") is True + assert environment.evaluate("'alpha' != 'beta'") is True + assert environment.evaluate("'alpha' < 'beta'") is True + try: + environment.evaluate("'alpha' < 1") + except ExecutionError as error: + assert error.error is not None + assert int(error.error.status) == 4 + assert error.error.type_name == "TypeError" + else: + raise AssertionError("mixed string/number ordering did not raise TypeError") + snapshot = environment.snapshot() + assert "__pyinbin_import__" not in snapshot.var assert snapshot.var["http_provider"] is game.provider.HttpProvider assert snapshot.var["input_value"] == 41.9 assert snapshot.var["floor_value"] == 41 assert snapshot.var["answer"] == 42 assert snapshot.var["nested"] == 42 assert snapshot.var["math"] is math + assert snapshot.var["imported_floor"] is math.floor + assert snapshot.var["unicode_text"] == "π" assert snapshot.var["game"] is game assert snapshot.var["add"] is add assert snapshot.var["tuple_roundtrip"] is tuple_roundtrip @@ -91,6 +134,9 @@ def main() -> int: "total": 42, "nested": {"value": 42}, } + assert snapshot.var["values"] == [40, 2] + assert snapshot.var["traced"] is True + assert snapshot.var["full_runtime_answer"] == 42 environment.execute( "answer = 7\n" @@ -117,6 +163,12 @@ def main() -> int: else: raise AssertionError("snapshot restore did not delete extra global") + environment.execute( + "import math\n" + "post_restore_import = math.floor(42.9)\n" + ) + assert environment.get("post_restore_import") == 42 + environment.remove("answer") environment.remove("answer", missing_ok=True) try: diff --git a/tests/native_full_core_probe_host.c b/tests/native_full_core_probe_host.c index 5d7b44b3..95516488 100644 --- a/tests/native_full_core_probe_host.c +++ b/tests/native_full_core_probe_host.c @@ -8,6 +8,7 @@ #define ABI_CALL __cdecl #else #include +#include #define LOAD_LIBRARY(path) dlopen((path), RTLD_NOW | RTLD_LOCAL) #define LOAD_SYMBOL(lib, name) dlsym((lib), (name)) #define ABI_CALL @@ -40,7 +41,14 @@ int main(int argc, char **argv) { fprintf(stderr, "full-core-parse=%lld\n", (long long)parsed_instructions); if (parsed_instructions <= 0) return 6; +#if !defined(_WIN32) + /* Preserve diagnostics if a native semantic regression loops forever. */ + alarm(20); +#endif int64_t result = probe(); +#if !defined(_WIN32) + alarm(0); +#endif printf("full-core=%lld\n", (long long)result); return result == 42 ? 0 : 7; } diff --git a/tests/native_function_host.c b/tests/native_function_host.c index 03e3d0d3..0c423dbf 100644 --- a/tests/native_function_host.c +++ b/tests/native_function_host.c @@ -1,7 +1,9 @@ +#define _GNU_SOURCE #include "portapy.h" #include #include +#include #include #if defined(_WIN32) @@ -9,13 +11,107 @@ #define LOAD_LIBRARY(path) ((void *)LoadLibraryA(path)) #define LOAD_SYMBOL(lib, name) ((void *)(uintptr_t)GetProcAddress((HMODULE)(lib), (name))) #define ABI_CALL __cdecl +static LONG WINAPI function_crash_filter(EXCEPTION_POINTERS *exception) { + DWORD code = 0; + void *address = NULL; + CONTEXT *context = NULL; + if (exception != NULL) { + context = exception->ContextRecord; + if (exception->ExceptionRecord != NULL) { + code = exception->ExceptionRecord->ExceptionCode; + address = exception->ExceptionRecord->ExceptionAddress; + } + } + HMODULE module = NULL; + char module_name[MAX_PATH] = {0}; + unsigned long long offset = 0; + if (address != NULL && GetModuleHandleExA( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCSTR)address, + &module + )) { + GetModuleFileNameA(module, module_name, (DWORD)sizeof(module_name)); + offset = (unsigned long long)((uintptr_t)address - (uintptr_t)module); + } + fprintf(stderr, + "function-crash: code=0x%08lx address=%p module=%s offset=0x%llx\n", + (unsigned long)code, + address, + module_name[0] == '\0' ? "" : module_name, + offset + ); +#if defined(_M_X64) || defined(__x86_64__) + if (context != NULL) { + fprintf(stderr, + "function-crash-context: rip=0x%llx rsp=0x%llx rbp=0x%llx\n", + (unsigned long long)context->Rip, + (unsigned long long)context->Rsp, + (unsigned long long)context->Rbp + ); + } +#endif + fflush(stderr); + return EXCEPTION_EXECUTE_HANDLER; +} +#define INSTALL_CRASH_HANDLER() SetUnhandledExceptionFilter(function_crash_filter) #else #include +#include +#include +#include #define LOAD_LIBRARY(path) dlopen((path), RTLD_NOW | RTLD_LOCAL) #define LOAD_SYMBOL(lib, name) dlsym((lib), (name)) #define ABI_CALL +static void function_signal_handler(int number, siginfo_t *info, void *context_pointer) { + uintptr_t instruction = 0; +#if defined(__x86_64__) && defined(REG_RIP) + ucontext_t *context = (ucontext_t *)context_pointer; + instruction = (uintptr_t)context->uc_mcontext.gregs[REG_RIP]; +#else + (void)context_pointer; +#endif + Dl_info module_info = {0}; + const char *module_name = ""; + uintptr_t offset = 0; + if (instruction != 0 && dladdr((void *)instruction, &module_info) != 0) { + if (module_info.dli_fname != NULL) module_name = module_info.dli_fname; + if (module_info.dli_fbase != NULL) { + offset = instruction - (uintptr_t)module_info.dli_fbase; + } + } + fprintf(stderr, + "function-crash: signal=%d fault=%p instruction=%p module=%s offset=0x%llx\n", + number, + info == NULL ? NULL : info->si_addr, + (void *)instruction, + module_name, + (unsigned long long)offset + ); + void *frames[32]; + int count = backtrace(frames, (int)(sizeof(frames) / sizeof(frames[0]))); + backtrace_symbols_fd(frames, count, 2); + fflush(stderr); + _Exit(128 + number); +} +static void install_function_crash_handler(void) { + struct sigaction action; + memset(&action, 0, sizeof(action)); + action.sa_sigaction = function_signal_handler; + action.sa_flags = SA_SIGINFO | SA_RESETHAND; + sigemptyset(&action.sa_mask); + sigaction(SIGSEGV, &action, NULL); + sigaction(SIGBUS, &action, NULL); + sigaction(SIGABRT, &action, NULL); +} +#define INSTALL_CRASH_HANDLER() install_function_crash_handler() #endif +#define TRACE_STEP(message) do { \ + fprintf(stderr, "function-step: %s\n", (message)); \ + fflush(stderr); \ +} while (0) + typedef portapy_status (ABI_CALL *initialize_fn)(void); typedef portapy_status (ABI_CALL *runtime_create_fn)(const portapy_config *, portapy_runtime *); typedef portapy_status (ABI_CALL *runtime_destroy_fn)(portapy_runtime); @@ -34,11 +130,9 @@ typedef portapy_status (ABI_CALL *release_fn)(portapy_runtime, portapy_value); static int execute(exec_fn function, portapy_runtime runtime, const char *source) { return function(runtime, (const uint8_t *)source, strlen(source), NULL, 0) == PORTAPY_OK; } - static int execute_status(exec_fn function, portapy_runtime runtime, const char *source, portapy_status expected) { return function(runtime, (const uint8_t *)source, strlen(source), NULL, 0) == expected; } - static int evaluate_i64(eval_fn function, as_i64_fn as_i64, release_fn release, portapy_runtime runtime, const char *source, int64_t expected) { portapy_value value = PORTAPY_NULL_VALUE; int64_t result = 0; @@ -46,7 +140,6 @@ static int evaluate_i64(eval_fn function, as_i64_fn as_i64, release_fn release, if (as_i64(runtime, value, &result) != PORTAPY_OK || result != expected) return 0; return release(runtime, value) == PORTAPY_OK; } - static int evaluate_bool(eval_fn function, as_bool_fn as_bool, release_fn release, portapy_runtime runtime, const char *source, int expected) { portapy_value value = PORTAPY_NULL_VALUE; int result = 0; @@ -54,7 +147,6 @@ static int evaluate_bool(eval_fn function, as_bool_fn as_bool, release_fn releas if (as_bool(runtime, value, &result) != PORTAPY_OK || result != expected) return 0; return release(runtime, value) == PORTAPY_OK; } - static int evaluate_status(eval_fn function, portapy_runtime runtime, const char *source, portapy_status expected) { portapy_value value = PORTAPY_NULL_VALUE; return function(runtime, (const uint8_t *)source, strlen(source), NULL, 0, &value) == expected; @@ -62,9 +154,11 @@ static int evaluate_status(eval_fn function, portapy_runtime runtime, const char int main(int argc, char **argv) { if (argc != 2) return 2; + INSTALL_CRASH_HANDLER(); + TRACE_STEP("load-library"); void *library = LOAD_LIBRARY(argv[1]); if (library == NULL) return 3; - + TRACE_STEP("resolve-symbols"); RESOLVE(initialize_fn, initialize, "portapy_library_initialize"); RESOLVE(runtime_create_fn, runtime_create, "portapy_runtime_create"); RESOLVE(runtime_destroy_fn, runtime_destroy, "portapy_runtime_destroy"); @@ -76,11 +170,13 @@ int main(int argc, char **argv) { RESOLVE(as_i64_fn, as_i64, "portapy_value_as_i64"); RESOLVE(release_fn, release, "portapy_value_release"); + TRACE_STEP("initialize"); if (initialize() != PORTAPY_OK) return 11; portapy_config config = {0}; config.struct_size = sizeof(config); config.abi_version = PORTAPY_ABI_VERSION; portapy_runtime runtime = PORTAPY_NULL_RUNTIME; + TRACE_STEP("runtime-create"); if (runtime_create(&config, &runtime) != PORTAPY_OK) return 12; const char first[] = @@ -90,22 +186,31 @@ int main(int argc, char **argv) { " total = left + right\n" " return total\n" "answer = add(20, 22)\n"; + TRACE_STEP("exec-first-functions"); if (!execute(exec_utf8, runtime, first)) return 13; + TRACE_STEP("eval-seven"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "seven()", 7)) return 14; + TRACE_STEP("eval-answer"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "answer", 42)) return 15; + TRACE_STEP("eval-add"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "add(3, 4)", 7)) return 16; portapy_value callable = PORTAPY_NULL_VALUE; portapy_value_kind kind = PORTAPY_VALUE_NONE; + TRACE_STEP("get-add-callable"); if (get_global(runtime, (const uint8_t *)"add", 3, &callable) != PORTAPY_OK) return 17; + TRACE_STEP("kind-add-callable"); if (get_kind(runtime, callable, &kind) != PORTAPY_OK || kind != PORTAPY_VALUE_CALLABLE) return 18; + TRACE_STEP("release-add-callable"); if (release(runtime, callable) != PORTAPY_OK) return 19; const char nested[] = "def double(value):\n" " return value * 2\n" "nested_answer = add(double(10), double(11))\n"; + TRACE_STEP("exec-nested-functions"); if (!execute(exec_utf8, runtime, nested)) return 20; + TRACE_STEP("eval-nested-answer"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "nested_answer", 42)) return 21; const char control[] = @@ -126,9 +231,13 @@ int main(int argc, char **argv) { "control_zero = classify(0)\n" "control_small = classify(3)\n" "control_large = classify(5)\n"; + TRACE_STEP("exec-function-control"); if (!execute(exec_utf8, runtime, control)) return 22; + TRACE_STEP("eval-control-zero"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "control_zero", 100)) return 23; + TRACE_STEP("eval-control-small"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "control_small", 4)) return 24; + TRACE_STEP("eval-control-large"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "control_large", 8)) return 25; const char arguments[] = @@ -138,14 +247,23 @@ int main(int argc, char **argv) { "mixed = combine(10, scale=4)\n" "reordered = combine(scale=2, left=18, right=3)\n" "nested_default = combine(combine(1), scale=2)\n"; + TRACE_STEP("exec-default-arguments"); if (!execute(exec_utf8, runtime, arguments)) return 26; + TRACE_STEP("eval-defaulted"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "defaulted", 42)) return 27; + TRACE_STEP("eval-mixed-keyword"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "mixed", 48)) return 28; + TRACE_STEP("eval-reordered-keyword"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "reordered", 42)) return 29; + TRACE_STEP("eval-nested-default"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "nested_default", 22)) return 30; + TRACE_STEP("error-missing-arguments"); if (!evaluate_status(eval_utf8, runtime, "combine()", PORTAPY_TYPE_ERROR)) return 31; + TRACE_STEP("error-duplicate-argument"); if (!evaluate_status(eval_utf8, runtime, "combine(1, left=2)", PORTAPY_TYPE_ERROR)) return 32; + TRACE_STEP("error-unknown-keyword"); if (!evaluate_status(eval_utf8, runtime, "combine(1, unknown=2)", PORTAPY_TYPE_ERROR)) return 33; + TRACE_STEP("error-positional-after-keyword"); if (!evaluate_status(eval_utf8, runtime, "combine(left=1, 2)", PORTAPY_COMPILE_ERROR)) return 34; const char capture[] = @@ -157,8 +275,11 @@ int main(int argc, char **argv) { "explicit_result = captured(7)\n" "def stable(value=6):\n" " return value\n"; + TRACE_STEP("exec-default-capture"); if (!execute(exec_utf8, runtime, capture)) return 35; + TRACE_STEP("eval-captured-result"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "captured_result", 10)) return 36; + TRACE_STEP("eval-explicit-result"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "explicit_result", 14)) return 37; const char redefine[] = @@ -166,13 +287,17 @@ int main(int argc, char **argv) { "def captured(value=seed):\n" " return value + 1\n" "seed = 100\n"; + TRACE_STEP("exec-redefinition"); if (!execute(exec_utf8, runtime, redefine)) return 38; + TRACE_STEP("eval-redefined-captured"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "captured()", 10)) return 39; const char invalid_redefinition[] = "def stable(value=missing_default):\n" " return value + 100\n"; + TRACE_STEP("exec-invalid-redefinition"); if (!execute_status(exec_utf8, runtime, invalid_redefinition, PORTAPY_NOT_FOUND)) return 40; + TRACE_STEP("eval-stable-after-invalid"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "stable()", 6)) return 41; const char parameter_kinds[] = @@ -188,28 +313,50 @@ int main(int argc, char **argv) { "marker_mixed = route(18, right=3, scale=2)\n" "required_result = required(40, offset=2)\n" "marker_captured = marker_capture()\n"; + TRACE_STEP("exec-parameter-kinds"); if (!execute(exec_utf8, runtime, parameter_kinds)) return 42; + TRACE_STEP("eval-qualified"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "qualified", 48)) return 43; + TRACE_STEP("eval-marker-mixed"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "marker_mixed", 42)) return 44; + TRACE_STEP("eval-required-result"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "required_result", 42)) return 45; + TRACE_STEP("eval-marker-captured"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "marker_captured", 5)) return 46; + TRACE_STEP("error-positional-only-keyword"); if (!evaluate_status(eval_utf8, runtime, "route(left=10)", PORTAPY_TYPE_ERROR)) return 47; + TRACE_STEP("error-too-many-positionals"); if (!evaluate_status(eval_utf8, runtime, "route(10, 2, 4)", PORTAPY_TYPE_ERROR)) return 48; + TRACE_STEP("error-required-keyword-only"); if (!evaluate_status(eval_utf8, runtime, "required(40)", PORTAPY_TYPE_ERROR)) return 49; + TRACE_STEP("error-keyword-only-positional"); if (!evaluate_status(eval_utf8, runtime, "required(40, 2)", PORTAPY_TYPE_ERROR)) return 50; + TRACE_STEP("error-invalid-double-star-parameter"); if (!execute_status(exec_utf8, runtime, "def bad(**):\n return 1\n", PORTAPY_COMPILE_ERROR)) return 51; + TRACE_STEP("tuple-index-first"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "(1, 2, 3)[0]", 1)) return 52; + TRACE_STEP("tuple-index-negative"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "(1, 2, 3)[-1]", 3)) return 53; + TRACE_STEP("tuple-index-nested"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "(1, (2, 3))[1][0]", 2)) return 54; + TRACE_STEP("tuple-len-empty"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "len(())", 0)) return 55; + TRACE_STEP("tuple-len-three"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "len((1, 2, 3))", 3)) return 56; + TRACE_STEP("unicode-len"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "len(\"\xC3\xA9\")", 1)) return 57; + TRACE_STEP("tuple-truth-empty"); if (!evaluate_bool(eval_utf8, as_bool, release, runtime, "not ()", 1)) return 58; + TRACE_STEP("tuple-truth-nonempty"); if (!evaluate_bool(eval_utf8, as_bool, release, runtime, "not (1,)", 0)) return 59; + TRACE_STEP("tuple-equality"); if (!evaluate_bool(eval_utf8, as_bool, release, runtime, "(1, (2, 3)) == (1, (2, 3))", 1)) return 60; + TRACE_STEP("tuple-inequality"); if (!evaluate_bool(eval_utf8, as_bool, release, runtime, "(1, 2) != (1, 3)", 1)) return 61; + TRACE_STEP("tuple-oob-error"); if (!evaluate_status(eval_utf8, runtime, "(1,)[2]", PORTAPY_RUNTIME_ERROR)) return 62; + TRACE_STEP("tuple-type-error"); if (!evaluate_status(eval_utf8, runtime, "(1,)[\"x\"]", PORTAPY_TYPE_ERROR)) return 63; const char tuples[] = @@ -219,14 +366,21 @@ int main(int argc, char **argv) { " return 0\n" "tuple_answer = summarize((18, 1, 20))\n" "tuple_empty = summarize(())\n"; + TRACE_STEP("exec-tuple-function"); if (!execute(exec_utf8, runtime, tuples)) return 64; + TRACE_STEP("eval-tuple-answer"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "tuple_answer", 41)) return 65; + TRACE_STEP("eval-tuple-empty"); if (!evaluate_i64(eval_utf8, as_i64, release, runtime, "tuple_empty", 0)) return 66; portapy_value missing = PORTAPY_NULL_VALUE; + TRACE_STEP("check-local-total-hidden"); if (get_global(runtime, (const uint8_t *)"total", 5, &missing) != PORTAPY_NOT_FOUND) return 67; + TRACE_STEP("check-local-current-hidden"); if (get_global(runtime, (const uint8_t *)"current", 7, &missing) != PORTAPY_NOT_FOUND) return 68; + TRACE_STEP("runtime-destroy"); if (runtime_destroy(runtime) != PORTAPY_OK) return 69; + TRACE_STEP("complete"); puts("native-functions: ok"); return 0; } diff --git a/tests/test_build_native_host_calls.py b/tests/test_build_native_host_calls.py new file mode 100644 index 00000000..c95ccd83 --- /dev/null +++ b/tests/test_build_native_host_calls.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from pathlib import Path + +from tools.build_native_host_calls import _linux_link_command + + +def test_linux_shared_library_links_its_math_dependency() -> None: + command = _linux_link_command( + gcc="gcc", + objects=["runtime.o", "glue.o"], + version_script=Path("portapy.map"), + output=Path("libportapy.so"), + ) + + assert command == [ + "gcc", + "-shared", + "runtime.o", + "glue.o", + "-Wl,--version-script=portapy.map", + "-lm", + "-o", + "libportapy.so", + ] diff --git a/tests/test_elf_pic.py b/tests/test_elf_pic.py index afe58c5e..718618e5 100644 --- a/tests/test_elf_pic.py +++ b/tests/test_elf_pic.py @@ -22,6 +22,7 @@ def test_external_calls_use_plt_and_data_uses_got() -> None: assert "mov rdx, [rel stdin wrt ..got]" in rewritten assert "mov rdx, [rdx]" in rewritten assert "section .note.GNU-stack noalloc noexec nowrite progbits" in rewritten + assert make_elf_pic(rewritten) == rewritten def test_local_calls_are_unchanged() -> None: @@ -40,6 +41,104 @@ def test_local_calls_are_unchanged() -> None: assert "call local_helper wrt ..plt" not in rewritten +def test_transient_push_is_aligned_around_call() -> None: + source = """\ +BITS 64 +section .text +probe: + push rbp + mov rbp, rsp + push rax + call helper + pop rax + leave + ret +helper: + ret +""" + rewritten = make_elf_pic(source) + assert ( + " push rax\n" + " sub rsp, 8\n" + " call helper\n" + " add rsp, 8\n" + " pop rax" + ) in rewritten + + +def test_stack_argument_reservation_absorbs_alignment_padding() -> None: + source = """\ +BITS 64 +section .text +probe: + push rbp + mov rbp, rsp + push rax + sub rsp, 16 + mov qword [rsp+0], 1 + mov qword [rsp+8], 2 + call helper + add rsp, 16 + pop rax + leave + ret +helper: + ret +""" + rewritten = make_elf_pic(source) + assert "sub rsp, 24" in rewritten + assert "add rsp, 24" in rewritten + assert "sub rsp, 8\n call helper" not in rewritten + + +def test_branch_alternatives_keep_aligned_calls_unchanged() -> None: + source = """\ +BITS 64 +section .text +probe: + push rbp + mov rbp, rsp + test rax, rax + jz .alternate + call first + leave + ret +.alternate: + call second + leave + ret +first: + ret +second: + ret +""" + rewritten = make_elf_pic(source) + assert "sub rsp, 8" not in rewritten + assert "call first" in rewritten + assert "call second" in rewritten + + +def test_ambiguous_stack_alignment_fails_closed() -> None: + source = """\ +BITS 64 +section .text +probe: + push rbp + mov rbp, rsp + test rax, rax + jz .joined + push rax +.joined: + call helper + leave + ret +helper: + ret +""" + with pytest.raises(ValueError, match="ambiguous stack alignment"): + make_elf_pic(source) + + def test_unknown_external_memory_reference_fails_closed() -> None: source = """\ BITS 64 diff --git a/tests/test_full_core_probe.py b/tests/test_full_core_probe.py index f4565273..e44c192c 100644 --- a/tests/test_full_core_probe.py +++ b/tests/test_full_core_probe.py @@ -1,12 +1,58 @@ +from __future__ import annotations + +import importlib +from pathlib import Path +import sys + from tools.materialize_full_reference_entry import main as materialize_reference_entry from tools.normalize_full_reference_abi_helpers import ( main as normalize_reference_abi_helpers, ) +from tools.normalize_full_reference_runtime import ( + PATH as REFERENCE_RUNTIME_PATH, + main as normalize_reference_runtime, +) + + +_TEMPORARY_MODULES = ( + "portapy.native_full_core_probe", + "portapy.native_full_reference_entry", + "portapy.reference_api", +) +_NATIVE_ENTRY_PATH = ( + Path(__file__).parents[1] / "src" / "portapy" / "native_full_reference_entry.py" +) + + +def _unload_temporary_modules() -> None: + for name in _TEMPORARY_MODULES: + sys.modules.pop(name, None) + importlib.invalidate_caches() def test_full_core_probe_executes_reference_abi_path() -> None: - materialize_reference_entry() - normalize_reference_abi_helpers() - from portapy.native_full_core_probe import portapy_full_core_probe + original_reference_runtime = REFERENCE_RUNTIME_PATH.read_text(encoding="utf-8") + original_native_entry = _NATIVE_ENTRY_PATH.read_text(encoding="utf-8") + try: + _unload_temporary_modules() + normalize_reference_runtime() + materialize_reference_entry() + normalize_reference_abi_helpers() + _unload_temporary_modules() + from portapy.native_full_core_probe import portapy_full_core_probe + from portapy.native_full_reference_entry import _runtimes - assert portapy_full_core_probe() == 42 + result = portapy_full_core_probe() + errors = [ + runtime.last_error() + for runtime in _runtimes + if runtime is not None and runtime.last_error() is not None + ] + assert result == 42, errors + finally: + REFERENCE_RUNTIME_PATH.write_text( + original_reference_runtime, + encoding="utf-8", + ) + _NATIVE_ENTRY_PATH.write_text(original_native_entry, encoding="utf-8") + _unload_temporary_modules() diff --git a/tests/test_full_core_workflow.py b/tests/test_full_core_workflow.py new file mode 100644 index 00000000..2c0e9179 --- /dev/null +++ b/tests/test_full_core_workflow.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +WORKFLOW = Path(__file__).parents[1] / ".github" / "workflows" / "native-full-core-probe.yml" + + +def test_full_core_workflow_runs_complete_normalizer_once_per_platform() -> None: + source = WORKFLOW.read_text(encoding="utf-8") + + assert source.count("python -m tools.normalize_full_core_validation") == 2 + assert "python tools/normalize_full_core_validation.py" not in source + for obsolete_command in ( + "python tools/normalize_full_core_probe.py", + "python tools/normalize_full_core_lambdas.py", + "python tools/normalize_full_core_native_semantics.py", + "python tools/normalize_full_core_opcode_maps.py", + ): + assert obsolete_command not in source + + +def test_full_core_normalizer_is_importable_as_a_module() -> None: + assert importlib.util.find_spec("tools.normalize_full_core_validation") is not None diff --git a/tests/test_full_reference_abi_normalization.py b/tests/test_full_reference_abi_normalization.py new file mode 100644 index 00000000..1203b471 --- /dev/null +++ b/tests/test_full_reference_abi_normalization.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import materialize_full_reference_entry as materializer +from tools import normalize_full_reference_abi_helpers as normalizer + + +def _function(module: ast.Module, name: str) -> ast.FunctionDef: + return next( + node + for node in module.body + if isinstance(node, ast.FunctionDef) and node.name == name + ) + + +def _source(module: ast.Module, name: str) -> str: + return ast.unparse(_function(module, name)) + + +def test_full_reference_normalization_installs_runtime_support( + tmp_path: Path, + monkeypatch, +) -> None: + output = tmp_path / "native_full_reference_entry.py" + monkeypatch.setattr(materializer, "OUTPUT", output) + monkeypatch.setattr(normalizer, "PATH", output) + + assert materializer.main() == 0 + assert normalizer.main() == 0 + + source = output.read_text(encoding="utf-8") + module = ast.parse(source) + classes = { + node.name + for node in module.body + if isinstance(node, ast.ClassDef) + } + assert "_PortaPyImportLoader" in classes + assert "source_size > len(source)" not in source + assert "_set_status(PORTAPY_" not in source + assert "status.value" not in source + assert "kind.value" not in source + + runtime_source = _source(module, "_portapy_runtime_create_impl") + assert "instance._vm._seed_builtins(instance._globals)" in runtime_source + assert "_PortaPyImportLoader(instance)" in runtime_source + assert "__pyinbin_import__" in runtime_source + assert "_set_status(Status.OK)" in runtime_source + + set_status_source = _source(module, "_set_status") + assert "_native_status_code(status)" in set_status_source + assert "int(status)" not in set_status_source + + status_code_source = _source(module, "_native_status_code") + assert "status is Status.OK" in status_code_source + assert "return PORTAPY_OK" in status_code_source + assert "status.value" not in status_code_source + + value_kind_code_source = _source(module, "_native_value_kind_code") + assert "kind is ValueKind.NONE" in value_kind_code_source + assert "return PORTAPY_VALUE_NONE" in value_kind_code_source + assert "kind.value" not in value_kind_code_source + + kind_source = _source(module, "_portapy_value_get_kind_impl") + assert "instance.value_kind(value)" in kind_source + assert "return _native_value_kind_code(kind)" in kind_source + assert "int(kind)" not in kind_source + assert "instance.unbox(value)" not in kind_source + assert "_value_kind(" not in kind_source + + bool_source = _source(module, "_portapy_value_as_bool_impl") + assert "instance.value_kind(value)" in bool_source + assert "kind is not ValueKind.BOOL" in bool_source + assert "instance.unbox(value)" in bool_source + assert "type(target)" not in bool_source + assert "PORTAPY_" not in bool_source + + list_source = _source(module, "_portapy_list_begin_impl") + assert "_set_status_code(PORTAPY_OK)" in list_source + assert "_set_status(PORTAPY_OK)" not in list_source + + dispatch_source = _source(module, "_portapy_host_dispatch_complete_impl") + assert "_set_status_code(status)" in dispatch_source + assert "_set_status(status)" not in dispatch_source + + assert "instance._store(_DataBuilder(kind, size), kind)" in _source( + module, "_portapy_value_from_data_begin_impl" + ) + tagged_stores = { + "_portapy_value_from_host_object_impl": "ValueKind.OBJECT", + "_portapy_value_from_host_callable_impl": "ValueKind.CALLABLE", + "_portapy_tuple_begin_impl": "ValueKind.TUPLE", + "_portapy_dict_begin_impl": "ValueKind.DICT", + "_portapy_list_begin_impl": "ValueKind.LIST", + } + for function_name, kind in tagged_stores.items(): + function_source = _source(module, function_name) + assert "instance._store(" in function_source + assert kind in function_source + + loader = next( + node + for node in module.body + if isinstance(node, ast.ClassDef) + and node.name == "_PortaPyImportLoader" + ) + loader_source = ast.unparse(loader) + assert "self.instance.read_global(parts[0])" in loader_source + assert "getattr(value, parts[index])" in loader_source + assert "raise ImportError(name)" in loader_source + assert "ModuleNotFoundError" not in loader_source diff --git a/tests/test_full_reference_container_access.py b/tests/test_full_reference_container_access.py new file mode 100644 index 00000000..351a492d --- /dev/null +++ b/tests/test_full_reference_container_access.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +from tools import materialize_full_reference_entry as materializer +from tools import normalize_full_reference_abi_helpers as abi_normalizer +from tools import normalize_full_reference_container_access as container_normalizer + + +def _function_source(module: ast.Module, name: str) -> str: + function = next( + node + for node in module.body + if isinstance(node, ast.FunctionDef) and node.name == name + ) + return ast.unparse(function) + + +def test_container_lengths_cross_typed_native_boundaries( + tmp_path: Path, + monkeypatch, +) -> None: + output = tmp_path / "native_full_reference_entry.py" + monkeypatch.setattr(materializer, "OUTPUT", output) + monkeypatch.setattr(abi_normalizer, "PATH", output) + monkeypatch.setattr(container_normalizer, "PATH", output) + + assert materializer.main() == 0 + assert abi_normalizer.main() == 0 + assert container_normalizer.main() == 0 + + module = ast.parse(output.read_text(encoding="utf-8")) + sequence_functions = ( + "_portapy_tuple_set_item_impl", + "_portapy_tuple_get_size_impl", + "_portapy_tuple_get_item_impl", + "_portapy_list_get_size_impl", + "_portapy_list_get_item_impl", + "_portapy_list_set_item_impl", + ) + for name in sequence_functions: + source = _function_source(module, name) + assert "_native_sequence_size(" in source + assert "len(" not in source + + dict_source = _function_source(module, "_portapy_dict_get_size_impl") + assert "_native_dict_size(target)" in dict_source + assert "len(" not in dict_source + + sequence_helper = _function_source(module, "_native_sequence_size") + assert "values: list[object]" in sequence_helper + assert "return len(values)" in sequence_helper + + dict_helper = _function_source(module, "_native_dict_size") + assert "values: dict[str, object]" in dict_helper + assert "return len(values)" in dict_helper + + with pytest.raises(RuntimeError, match="already installed"): + container_normalizer.main() diff --git a/tests/test_nasm_direct_float_bits.py b/tests/test_nasm_direct_float_bits.py new file mode 100644 index 00000000..31c050b8 --- /dev/null +++ b/tests/test_nasm_direct_float_bits.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import pytest + +from tools.nasm_direct_float_abi import append_direct_float_abi + + +_SOURCE = ''' +_portapy_last_status_impl: + ret +_portapy_value_from_f64_bits_impl: + ret +_portapy_value_as_f64_bits_impl: + ret +''' + + +def test_linux_float_wrappers_move_ieee_bits_through_gprs() -> None: + result = append_direct_float_abi(_SOURCE, target="linux") + assert "movq rsi, xmm0" in result + assert "call _portapy_value_from_f64_bits_impl" in result + assert "call _portapy_value_as_f64_bits_impl" in result + assert "movsd" not in result + assert "mov rcx, [rsp + 8]" in result + assert "mov [rdx], rcx" in result + assert "mov [rdx], rax" not in result + + +def test_windows_float_wrappers_move_ieee_bits_through_gprs() -> None: + result = append_direct_float_abi(_SOURCE, target="windows") + assert "movq rdx, xmm1" in result + assert "call _portapy_value_from_f64_bits_impl" in result + assert "call _portapy_value_as_f64_bits_impl" in result + assert "movsd" not in result + assert "mov [r8], r9" in result + + +def test_rejects_old_float_implementation_labels() -> None: + with pytest.raises(ValueError, match="f64_bits"): + append_direct_float_abi( + "_portapy_last_status_impl:\n_portapy_value_from_f64_impl:\n" + "_portapy_value_as_f64_impl:\n", + target="linux", + ) diff --git a/tests/test_nasm_exception_handlers.py b/tests/test_nasm_exception_handlers.py new file mode 100644 index 00000000..a36e1bd2 --- /dev/null +++ b/tests/test_nasm_exception_handlers.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import pytest + +from tools.nasm_exception_handlers import restore_exception_handler_epilogues + + +def test_restores_active_handler_without_clobbering_rax() -> None: + source = """\ +section .text +probe: + push rbp + mov rbp, rsp + sub rsp, 240 + mov rax, [rel _runtime_handler_top] + mov [rbp-208], rax + lea rax, [rbp-200] + mov [rel _runtime_handler_top], rax + lea rax, [rbp-200] + call _runtime_setjmp + mov rax, 42 +.Lret_probe: + mov rsp, rbp + pop rbp + ret +""" + rewritten, functions, epilogues = restore_exception_handler_epilogues(source) + assert (functions, epilogues) == (1, 1) + assert "mov r10, [rel _runtime_handler_top]" in rewritten + assert "lea r11, [rbp-200]" in rewritten + assert "mov r10, [rbp-208]" in rewritten + assert "mov [rel _runtime_handler_top], r10" in rewritten + assert "mov rax, 42" in rewritten + assert restore_exception_handler_epilogues(rewritten)[0] == rewritten + + +def test_nested_handlers_are_peeled_inner_first() -> None: + source = """\ +section .text +probe: + push rbp + mov rbp, rsp + sub rsp, 448 + mov rax, [rel _runtime_handler_top] + mov [rbp-208], rax + lea rax, [rbp-200] + mov [rel _runtime_handler_top], rax + lea rax, [rbp-200] + call _runtime_setjmp + mov rax, [rel _runtime_handler_top] + mov [rbp-416], rax + lea rax, [rbp-408] + mov [rel _runtime_handler_top], rax + lea rax, [rbp-408] + call _runtime_setjmp + mov rsp, rbp + pop rbp + ret +""" + rewritten, _, _ = restore_exception_handler_epilogues(source) + assert rewritten.index("lea r11, [rbp-408]") < rewritten.index( + "lea r11, [rbp-200]" + ) + + +def test_malformed_setjmp_setup_fails_closed() -> None: + source = """\ +section .text +probe: + push rbp + mov rbp, rsp + call _runtime_setjmp + mov rsp, rbp + pop rbp + ret +""" + with pytest.raises(ValueError, match="setjmp buffer"): + restore_exception_handler_epilogues(source) diff --git a/tests/test_native_internal_globals.py b/tests/test_native_internal_globals.py new file mode 100644 index 00000000..f354c661 --- /dev/null +++ b/tests/test_native_internal_globals.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from portapy.native_internal_globals import _is_public_global + + +def test_runtime_internal_globals_are_hidden() -> None: + assert not _is_public_global("__pyinbin_import__") + assert not _is_public_global("__pyinbin_future_state") + assert not _is_public_global("__portapy_internal_cache") + + +def test_normal_dunder_and_user_globals_remain_public() -> None: + assert _is_public_global("__name__") + assert _is_public_global("answer") + assert _is_public_global("_private_user_value") diff --git a/tests/test_native_object_binary.py b/tests/test_native_object_binary.py new file mode 100644 index 00000000..55856782 --- /dev/null +++ b/tests/test_native_object_binary.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from portapy import NativeObjectReference +from portapy import native_binary as native +from portapy.reference_api import Status, ValueKind + + +class FakeObjectApi: + def __init__(self, *, host_id: int | None) -> None: + self.host_id = host_id + self.cleared = 0 + + def portapy_value_get_kind(self, runtime, handle, out_kind) -> int: + del runtime, handle + out_kind._obj.value = int(ValueKind.OBJECT) + return int(Status.OK) + + def portapy_value_get_host_id(self, runtime, handle, out_host_id) -> int: + del runtime, handle + if self.host_id is None: + return int(Status.TYPE_ERROR) + out_host_id._obj.value = self.host_id + return int(Status.OK) + + def portapy_error_clear(self, runtime) -> int: + del runtime + self.cleared += 1 + return int(Status.OK) + + +def environment_for(api: FakeObjectApi) -> native.NativeEnvironment: + environment = object.__new__(native.NativeEnvironment) + environment._api = api + environment._runtime = native._U64(1) + environment._closed = False + environment._objects = {} + return environment + + +def test_vm_owned_object_returns_opaque_reference() -> None: + api = FakeObjectApi(host_id=None) + environment = environment_for(api) + result = environment._unbox(99) + assert isinstance(result, NativeObjectReference) + assert api.cleared == 1 + + +def test_registered_host_object_still_resolves_to_python_object() -> None: + api = FakeObjectApi(host_id=42) + environment = environment_for(api) + expected = object() + environment._objects[42] = expected + assert environment._unbox(99) is expected + assert api.cleared == 0 diff --git a/tests/test_normalize_full_core_boolops.py b/tests/test_normalize_full_core_boolops.py new file mode 100644 index 00000000..c1fded5b --- /dev/null +++ b/tests/test_normalize_full_core_boolops.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_core_boolops as normalizer + + +def test_rewrites_only_boolop_operands_to_dynamic_getattr( + tmp_path: Path, monkeypatch, +) -> None: + source = tmp_path / "native_ast.py" + source.write_text( + '''def _convert_expr(node, lifted): + if isinstance(node, A.BinOp): + return BinOp(_convert_expr(node.left, lifted), node.op, _convert_expr(node.right, lifted)) + if isinstance(node, A.BoolOp): + return BoolOp( + And() if node.op == "and" else Or(), + [_convert_expr(node.left, lifted), _convert_expr(node.right, lifted)], + ) + raise RuntimeError("unsupported") +''', + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", source) + + assert normalizer.main() == 0 + + module = ast.parse(source.read_text(encoding="utf-8")) + text = ast.unparse(module) + assert "_convert_expr(getattr(node, 'left'), lifted)" in text + assert "_convert_expr(getattr(node, 'right'), lifted)" in text + # The already-working BinOp path remains direct/static. + assert "_convert_expr(node.left, lifted), node.op" in text + + +def test_fails_closed_without_exact_boolop_shape(tmp_path: Path, monkeypatch) -> None: + source = tmp_path / "native_ast.py" + source.write_text( + '''def _convert_expr(node, lifted): + return node +''', + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", source) + + try: + normalizer.main() + except RuntimeError as error: + assert "expected 2 fields" in str(error) + else: + raise AssertionError("normalizer accepted a missing BoolOp path") diff --git a/tests/test_normalize_full_core_builtin_parameter_collisions.py b/tests/test_normalize_full_core_builtin_parameter_collisions.py new file mode 100644 index 00000000..0a30b68d --- /dev/null +++ b/tests/test_normalize_full_core_builtin_parameter_collisions.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools.normalize_full_core_builtin_parameter_collisions import normalize_tree + + +SOURCE = '''class ExceptHandler: + def __init__(self, type, name, body): + self.type = type + self.name = name + self.body = body + +class Subscript: + def __init__(self, value, slice, ctx=None): + self.value = value + self.slice = slice + self.ctx = ctx + +def build(value, index): + handler = ExceptHandler(type=value, name=None, body=[]) + subscript = Subscript(value=value, slice=index, ctx=None) + return handler, subscript +''' + + +def test_repairs_all_builtin_parameters(tmp_path: Path) -> None: + root = tmp_path / "portapy" + root.mkdir() + module = root / "module.py" + module.write_text(SOURCE, encoding="utf-8") + + constructors, calls, files = normalize_tree(root) + + assert (constructors, calls, files) == (2, 2, 1) + source = module.read_text(encoding="utf-8") + assert "def __init__(self, type_value, name, body):" in source + assert "self.type = type_value" in source + assert "ExceptHandler(type_value=value, name=None, body=[])" in source + assert "def __init__(self, value, slice_value, ctx=None):" in source + assert "self.slice = slice_value" in source + assert "Subscript(value=value, slice_value=index, ctx=None)" in source + ast.parse(source) + + +def test_positional_calls_remain_unchanged(tmp_path: Path) -> None: + root = tmp_path / "portapy" + root.mkdir() + module = root / "module.py" + module.write_text( + SOURCE.replace( + "ExceptHandler(type=value, name=None, body=[])", + "ExceptHandler(value, None, [])", + ).replace( + "Subscript(value=value, slice=index, ctx=None)", + "Subscript(value, index, None)", + ), + encoding="utf-8", + ) + + constructors, calls, files = normalize_tree(root) + + assert (constructors, calls, files) == (2, 0, 1) + source = module.read_text(encoding="utf-8") + assert "ExceptHandler(value, None, [])" in source + assert "Subscript(value, index, None)" in source + assert "self.type = type_value" in source + assert "self.slice = slice_value" in source diff --git a/tests/test_normalize_full_core_collections.py b/tests/test_normalize_full_core_collections.py new file mode 100644 index 00000000..b449a25c --- /dev/null +++ b/tests/test_normalize_full_core_collections.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tools import normalize_full_core_collections as collections + + +def _run(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, source: str) -> str: + path = tmp_path / "vm.py" + path.write_text(source, encoding="utf-8") + monkeypatch.setattr(collections, "PATH", path) + assert collections.main() == 0 + return path.read_text(encoding="utf-8") + + +def _assert_restored(result: str) -> None: + assert "frame.stack.append(None)" not in result + assert "frame.stack.append(tuple(values))" in result + assert "frame.stack.append(set(values))" in result + + +def test_restores_canonical_collection_bootstrap( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _assert_restored( + _run(tmp_path, monkeypatch, collections._CANONICAL_BOOTSTRAP) + ) + + +def test_restores_compact_collection_bootstrap( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + result = _run(tmp_path, monkeypatch, collections._COMPACT_BOOTSTRAP) + _assert_restored(result) + assert "if instr.arg: del frame.stack[-instr.arg:]" in result + + +def test_rejects_ambiguous_collection_bootstrap( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "vm.py" + path.write_text( + collections._CANONICAL_BOOTSTRAP + collections._COMPACT_BOOTSTRAP, + encoding="utf-8", + ) + monkeypatch.setattr(collections, "PATH", path) + with pytest.raises(RuntimeError, match="expected one BUILD_TUPLE/BUILD_SET block"): + collections.main() + + +def test_rejects_collection_block_without_placeholder( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = collections._CANONICAL_BOOTSTRAP.replace( + "frame.stack.append(None)", "frame.stack.append(values)" + ) + path = tmp_path / "vm.py" + path.write_text(source, encoding="utf-8") + monkeypatch.setattr(collections, "PATH", path) + with pytest.raises(RuntimeError, match="placeholder collection result"): + collections.main() diff --git a/tests/test_normalize_full_core_compiler_symbols.py b/tests/test_normalize_full_core_compiler_symbols.py new file mode 100644 index 00000000..dbf7190a --- /dev/null +++ b/tests/test_normalize_full_core_compiler_symbols.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from tools import normalize_full_core_lambdas as normalizer + + +def _sema_source() -> str: + return '''BUILTINS = { + "repr": (1, 1), + "type": (1, 1), +} +BUILTIN_EXCEPTIONS = frozenset({ + "FileNotFoundError", +}) +BUILTIN_TYPE_NAMES = frozenset({ + "int", "float", "str", "bool", "list", "dict", "tuple", "set", +}) +''' + + +def _codegen_source() -> str: + return '''BUILTIN_EXC_PARENTS = { + "StopIteration": "Exception", +} +BUILTIN_EXC_IDS = { + "StopIteration": 21, + "IOError": 19, # alias for OSError (same id) +} +BUILTIN_TYPE_IDS = { + "set": -8, +} +''' + + +def test_enables_all_portapy_compiler_symbols() -> None: + sema, codegen = normalizer._patch_compiler_runtime_symbols( + _sema_source(), _codegen_source() + ) + for name, parent in normalizer._PORTAPY_EXCEPTION_PARENTS.items(): + assert f' "{name}",' in sema + assert f' "{name}": {parent!r},' in codegen + for name, identifier in normalizer._PORTAPY_EXCEPTION_IDS.items(): + assert f' "{name}": {identifier},' in codegen + for name, identifier in normalizer._PORTAPY_TYPE_IDS.items(): + assert f' "{name}",' in sema + assert f' "{name}": {identifier},' in codegen + + +def test_compiler_symbol_patch_is_idempotent() -> None: + sema, codegen = normalizer._patch_compiler_runtime_symbols( + _sema_source(), _codegen_source() + ) + assert normalizer._patch_compiler_runtime_symbols(sema, codegen) == ( + sema, + codegen, + ) diff --git a/tests/test_normalize_full_core_default_expressions.py b/tests/test_normalize_full_core_default_expressions.py new file mode 100644 index 00000000..9186a8c5 --- /dev/null +++ b/tests/test_normalize_full_core_default_expressions.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_core_default_expressions as normalizer + + +SOURCE = '''class Parser: + def _parse_optional_default(self): + if not self._check("OP", "="): + return None + self._eat() + return self._parse_default_literal() +''' + + +def test_uses_full_expression_parser(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "native_ast.py" + path.write_text(SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + source = path.read_text(encoding="utf-8") + assert "return self._parse_expr()" in source + assert "return self._parse_default_literal()" not in source + ast.parse(source) + + +def test_fails_closed_when_parser_shape_changes( + tmp_path: Path, + monkeypatch, +) -> None: + path = tmp_path / "native_ast.py" + path.write_text( + SOURCE.replace("self._parse_default_literal()", "self._parse_expr()"), + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "restricted literal path" in str(error) + else: + raise AssertionError("normalizer accepted an unexpected parser shape") diff --git a/tests/test_normalize_full_core_exception_statuses.py b/tests/test_normalize_full_core_exception_statuses.py new file mode 100644 index 00000000..f82e15b0 --- /dev/null +++ b/tests/test_normalize_full_core_exception_statuses.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_core_exception_statuses as normalizer + + +VM_SOURCE = '''class VirtualMachine: + def _run_frame(self, frame): + while True: + try: + run() + except BaseException as exc: + exc = _NativeCaughtException(exc) + caught = False + if frame.handlers: + frame.stack.append(exc) + frame.active_exception = exc + caught = True + if caught: + continue + raise +''' + +REFERENCE_SOURCE = '''class Runtime: + def exec_utf8(self, source): + try: + self._vm.run(source) + except TypeError as error: + return self._capture_native(Status.TYPE_ERROR, "TypeError", "PortaPy type error") + except BaseException as error: + return self._capture_native(Status.RUNTIME_ERROR, "RuntimeError", "PortaPy runtime error") +''' + + +def test_preserves_escape_identity_and_maps_name_error( + tmp_path: Path, + monkeypatch, +) -> None: + vm_path = tmp_path / "vm.py" + reference_path = tmp_path / "reference_api.py" + vm_path.write_text(VM_SOURCE, encoding="utf-8") + reference_path.write_text(REFERENCE_SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "VM_PATH", vm_path) + monkeypatch.setattr(normalizer, "REFERENCE_PATH", reference_path) + + assert normalizer.main() == 0 + + vm_source = vm_path.read_text(encoding="utf-8") + assert "exc = _NativeCaughtException(exc)" not in vm_source + assert "native_exception = _NativeCaughtException(exc)" in vm_source + assert "frame.stack.append(native_exception)" in vm_source + assert "frame.active_exception = native_exception" in vm_source + assert "raise" in vm_source + + reference_source = reference_path.read_text(encoding="utf-8") + assert "except NameError as error:" in reference_source + assert "Status.NOT_FOUND" in reference_source + assert reference_source.index("except NameError") < reference_source.index( + "except BaseException" + ) + ast.parse(vm_source) + ast.parse(reference_source) + + +def test_fails_closed_without_wrapper_assignment( + tmp_path: Path, + monkeypatch, +) -> None: + vm_path = tmp_path / "vm.py" + reference_path = tmp_path / "reference_api.py" + vm_path.write_text( + VM_SOURCE.replace( + " exc = _NativeCaughtException(exc)\n", + "", + ), + encoding="utf-8", + ) + reference_path.write_text(REFERENCE_SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "VM_PATH", vm_path) + monkeypatch.setattr(normalizer, "REFERENCE_PATH", reference_path) + + try: + normalizer.main() + except RuntimeError as error: + assert "wrapper handler" in str(error) + else: + raise AssertionError("normalizer accepted a missing exception wrapper") diff --git a/tests/test_normalize_full_core_expr_stmt_assembly.py b/tests/test_normalize_full_core_expr_stmt_assembly.py new file mode 100644 index 00000000..627b2c66 --- /dev/null +++ b/tests/test_normalize_full_core_expr_stmt_assembly.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from tools.normalize_full_core_expr_stmt_assembly import ( + fix_expr_stmt_initializer_assembly, +) + + +LINUX = '''_npr_ast_nodes_ExprStmt____init__: + push rbp + mov rbp, rsp + mov [rbp-8], rdi + mov [rbp-16], rsi + mov [rbp-24], rdx + mov rax, 126 + push rax + ret +_npr_ast_nodes_Pass____init__: + ret +''' + +WINDOWS = '''_npr_ast_nodes_ExprStmt____init__: + push rbp + mov rbp, rsp + mov [rbp-8], rcx + mov [rbp-16], rdx + mov [rbp-24], r8 + mov rax, 126 + push rax + ret +_npr_ast_nodes_Pass____init__: + ret +''' + + +def test_repairs_linux_parameter_load() -> None: + source, count = fix_expr_stmt_initializer_assembly(LINUX, target="linux") + assert count == 1 + assert "mov rax, 126" not in source + assert "mov rax, [rbp-16]" in source + + +def test_repairs_windows_parameter_load() -> None: + source, count = fix_expr_stmt_initializer_assembly(WINDOWS, target="windows") + assert count == 1 + assert "mov rax, 126" not in source + assert "mov rax, [rbp-16]" in source + + +def test_accepts_already_correct_parameter_load() -> None: + correct = LINUX.replace("mov rax, 126", "mov rax, [rbp-16]") + source, count = fix_expr_stmt_initializer_assembly(correct, target="linux") + + assert count == 0 + assert source == correct + + +def test_fails_closed_when_bad_load_changes() -> None: + try: + fix_expr_stmt_initializer_assembly( + LINUX.replace("mov rax, 126", "mov rax, 125"), + target="linux", + ) + except RuntimeError as error: + assert "neither the static dict-token load" in str(error) + else: + raise AssertionError("assembly transform accepted a changed bad load") + + +def test_fails_closed_when_parameter_spill_changes() -> None: + try: + fix_expr_stmt_initializer_assembly( + WINDOWS.replace("mov [rbp-16], rdx", "mov [rbp-32], rdx"), + target="windows", + ) + except RuntimeError as error: + assert "parameter spills changed" in str(error) + else: + raise AssertionError("assembly transform accepted changed parameter spills") diff --git a/tests/test_normalize_full_core_expr_stmt_initializer.py b/tests/test_normalize_full_core_expr_stmt_initializer.py new file mode 100644 index 00000000..531ecd22 --- /dev/null +++ b/tests/test_normalize_full_core_expr_stmt_initializer.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_core_expr_stmt_initializer as normalizer + + +SOURCE = '''@dataclass +class _npr_ast_nodes_ExprStmt: + expr: dict + pos: object + +class Other: + pass + +def build(value, pos): + return _npr_ast_nodes_ExprStmt(expr=value, pos=pos) +''' + + +def test_installs_noncolliding_explicit_initializer( + tmp_path: Path, + monkeypatch, +) -> None: + path = tmp_path / "native_ast.py" + path.write_text(SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + source = path.read_text(encoding="utf-8") + assert "def __init__(self, expr_value: dict, pos: dict) -> None:" in source + assert "self.expr = expr_value" in source + assert "self.pos = pos" in source + assert "_npr_ast_nodes_ExprStmt(expr_value=value, pos=pos)" in source + assert "values: list[dict]" not in source + ast.parse(source) + + +def test_positional_constructor_calls_remain_valid( + tmp_path: Path, + monkeypatch, +) -> None: + path = tmp_path / "native_ast.py" + path.write_text( + SOURCE.replace( + "_npr_ast_nodes_ExprStmt(expr=value, pos=pos)", + "_npr_ast_nodes_ExprStmt(value, pos)", + ), + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + source = path.read_text(encoding="utf-8") + assert "_npr_ast_nodes_ExprStmt(value, pos)" in source + ast.parse(source) + + +def test_fails_closed_without_expr_stmt_class(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "native_ast.py" + path.write_text("class Other:\n pass\n", encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "expected one class" in str(error) + else: + raise AssertionError("normalizer accepted a missing ExprStmt class") + + +def test_fails_closed_with_existing_initializer( + tmp_path: Path, + monkeypatch, +) -> None: + path = tmp_path / "native_ast.py" + path.write_text( + SOURCE.replace( + " pos: object\n", + " pos: object\n\n def __init__(self):\n pass\n", + ), + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "already has an initializer" in str(error) + else: + raise AssertionError("normalizer replaced an existing initializer") diff --git a/tests/test_normalize_full_core_function_binding.py b/tests/test_normalize_full_core_function_binding.py new file mode 100644 index 00000000..0f228479 --- /dev/null +++ b/tests/test_normalize_full_core_function_binding.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from pathlib import Path + +from tools import normalize_full_core_function_binding as normalizer + + +_SOURCE = '''class VirtualMachine: + def _call(self, target, args, kwargs=None): + positional = list(args[:total]) + locals_ = dict(zip(target.code.arg_names, positional)) + if target.code.kwarg_name: + locals_[target.code.kwarg_name] = { + name: value for name, value in kwargs.items() + if name in target.code.posonly_names or ( + name not in target.code.arg_names and name not in target.code.kwonly_names + ) + } + return locals_ +''' + + +def test_replaces_zip_and_kwargs_comprehension(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "vm.py" + path.write_text(_SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + source = path.read_text(encoding="utf-8") + assert "dict(zip(" not in source + assert "for name, value in kwargs.items()" not in source + assert "argument_names: list[str] = target.code.arg_names" in source + assert "argument_name: str = argument_names[bind_index]" in source + assert "locals_[argument_name] = positional[bind_index]" in source + assert "extra_kwargs[name] = kwargs[name]" in source + + +def test_is_idempotent(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "vm.py" + path.write_text(_SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + first = path.read_text(encoding="utf-8") + assert normalizer.main() == 0 + assert path.read_text(encoding="utf-8") == first + + +def test_fails_closed_when_binding_shape_changes(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "vm.py" + path.write_text("class VirtualMachine: pass\n", encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "source shape changed" in str(error) + else: + raise AssertionError("normalizer accepted an unknown function binding shape") diff --git a/tests/test_normalize_full_core_function_parameter_names.py b/tests/test_normalize_full_core_function_parameter_names.py new file mode 100644 index 00000000..28038c11 --- /dev/null +++ b/tests/test_normalize_full_core_function_parameter_names.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from pathlib import Path + +from tools import normalize_full_core_function_parameter_names as normalizer + + +_SPECIAL = ''' nested.posonly_names = [arg.arg for arg in node.args.posonlyargs] + nested.kwonly_names = [arg.arg for arg in node.args.kwonlyargs] + nested.vararg_name = node.args.vararg.arg if node.args.vararg else None + nested.kwarg_name = node.args.kwarg.arg if node.args.kwarg else None +''' + +_SOURCE = '''class _Lowerer: + def expr(self, node): + lambda_arguments = list(node.args.posonlyargs) + for argument in node.args.args: + lambda_arguments.append(argument) + nested = _Lowerer("", [arg.arg for arg in lambda_arguments]) +''' + _SPECIAL + ''' def stmt(self, node): + function_arguments = list(node.args.posonlyargs) + for argument in node.args.args: + function_arguments.append(argument) + nested = _Lowerer(node.name, [arg.arg for arg in function_arguments]) +''' + _SPECIAL + + +def test_replaces_opaque_parameter_extraction(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "frontend.py" + path.write_text(_SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + source = path.read_text(encoding="utf-8") + assert "[arg.arg for arg in function_arguments]" not in source + assert "[arg.arg for arg in lambda_arguments]" not in source + assert "[arg.arg for arg in node.args.posonlyargs]" not in source + assert "[arg.arg for arg in node.args.kwonlyargs]" not in source + assert 'function_argument_name: str = getattr(argument, "arg")' in source + assert 'lambda_argument_name: str = getattr(argument, "arg")' in source + assert source.count('positional_only_name: str = getattr(argument, "arg")') == 2 + assert source.count('keyword_only_name: str = getattr(argument, "arg")') == 2 + assert source.count('variadic_positional = getattr(node.args, "vararg")') == 2 + assert source.count('variadic_positional_name: str = getattr(variadic_positional, "arg")') == 2 + assert source.count('variadic_keyword = getattr(node.args, "kwarg")') == 2 + assert source.count('variadic_keyword_name: str = getattr(variadic_keyword, "arg")') == 2 + assert "function_argument_name: str = argument.arg" not in source + assert "lambda_argument_name: str = argument.arg" not in source + + +def test_is_idempotent(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "frontend.py" + path.write_text(_SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + first = path.read_text(encoding="utf-8") + assert normalizer.main() == 0 + assert path.read_text(encoding="utf-8") == first + + +def test_fails_closed_for_unknown_shape(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "frontend.py" + path.write_text("class _Lowerer: pass\n", encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "source shape changed" in str(error) + else: + raise AssertionError("normalizer accepted unknown parameter extraction") diff --git a/tests/test_normalize_full_core_function_specs.py b/tests/test_normalize_full_core_function_specs.py new file mode 100644 index 00000000..47758ce9 --- /dev/null +++ b/tests/test_normalize_full_core_function_specs.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from pathlib import Path + +from tools import normalize_full_core_function_specs as normalizer + + +SOURCE = '''class Lowerer: + def lower_lambda(self, node): + for default in node.args.defaults: + self.expr(default) + self.emit(Op.MAKE_FUNCTION, self.constant((nested.finish(), len(node.args.defaults), 0))) + + def lower_function(self, node): + for default in node.args.defaults: + self.expr(default) + kw_default_count = 0 + for default in node.args.kw_defaults: + if default is None: + continue + self.expr(default) + kw_default_count += 1 + self.emit( + Op.MAKE_FUNCTION, + self.constant((nested.finish(), len(node.args.defaults), kw_default_count, annotations)), + ) +''' + + +def test_counts_defaults_and_pins_ast_elements( + tmp_path: Path, monkeypatch, +) -> None: + path = tmp_path / "frontend.py" + path.write_text(SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + source = path.read_text(encoding="utf-8") + assert 'lambda_defaults: list[dict] = getattr(node.args, "defaults")' in source + assert "lambda_default: dict = lambda_defaults[lambda_default_index]" in source + assert "lambda_default_count += 1" in source + assert "(nested.finish(), lambda_default_count, 0, {})" in source + assert 'function_defaults: list[dict] = getattr(node.args, "defaults")' in source + assert "function_default: dict = function_defaults[default_index]" in source + assert "default_count += 1" in source + assert 'keyword_defaults: list[dict] = getattr(node.args, "kw_defaults")' in source + assert "keyword_default: dict = keyword_defaults[keyword_default_index]" in source + assert "(nested.finish(), default_count, kw_default_count, annotations)" in source + assert "len(node.args.defaults)" not in source + assert "for default in node.args.defaults:" not in source + assert "for default in node.args.kw_defaults:" not in source + + +def test_fails_closed_without_lambda_defaults_shape( + tmp_path: Path, monkeypatch, +) -> None: + path = tmp_path / "frontend.py" + path.write_text( + SOURCE.replace( + ''' for default in node.args.defaults: + self.expr(default) + self.emit(Op.MAKE_FUNCTION, self.constant((nested.finish(), len(node.args.defaults), 0))) +''', + " pass\n", + ), + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "lambda defaults" in str(error) + else: + raise AssertionError("normalizer accepted a missing lambda defaults producer") diff --git a/tests/test_normalize_full_core_keyword_calls.py b/tests/test_normalize_full_core_keyword_calls.py new file mode 100644 index 00000000..5fc535a7 --- /dev/null +++ b/tests/test_normalize_full_core_keyword_calls.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from pathlib import Path + +from tools import normalize_full_core_keyword_calls as normalizer + + +def test_frontend_keyword_fields_are_extracted_through_typed_locals( + tmp_path: Path, + monkeypatch, +) -> None: + frontend = tmp_path / "frontend.py" + frontend.write_text( + ''' for keyword in node.keywords: + self.expr(keyword.value) + names = tuple(keyword.arg for keyword in node.keywords) + self.emit(Op.CALL_KW, self.constant((tuple(arg_specs), names))) +''', + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "FRONTEND_PATH", frontend) + + normalizer._normalize_frontend() + + source = frontend.read_text(encoding="utf-8") + assert 'keyword_value: ast.expr = getattr(keyword, "value")' in source + assert 'keyword_name = getattr(keyword, "arg")' in source + assert "keyword_names.append(keyword_name)" in source + assert "self.expr(keyword.value)" not in source + assert "keyword.arg for keyword" not in source diff --git a/tests/test_normalize_full_core_len.py b/tests/test_normalize_full_core_len.py new file mode 100644 index 00000000..6c57deb4 --- /dev/null +++ b/tests/test_normalize_full_core_len.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_core_len as normalizer + + +LOADER_SOURCE = '''def _builtin_len(value: object) -> int: + return len(value) +''' + +FRONTEND_SOURCE = '''class _Lowerer: + def expr(self, node): + if isinstance(node, ast.Constant): + pass + elif isinstance(node, ast.Call) and not node.keywords and all(not isinstance(arg, ast.Starred) for arg in node.args): + self.expr(node.func) + for arg in node.args: + self.expr(arg) + self.emit(Op.CALL, len(node.args)) + elif isinstance(node, ast.Call): + pass +''' + + +def test_threads_expression_kind_into_native_len( + tmp_path: Path, + monkeypatch, +) -> None: + loader = tmp_path / "loader.py" + frontend = tmp_path / "frontend.py" + loader.write_text(LOADER_SOURCE, encoding="utf-8") + frontend.write_text(FRONTEND_SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "LOADER_PATH", loader) + monkeypatch.setattr(normalizer, "FRONTEND_PATH", frontend) + + assert normalizer.main() == 0 + + loader_source = loader.read_text(encoding="utf-8") + assert "def _builtin_len(value: object, kind: int=0) -> int:" in loader_source + assert "text_value: str = value" in loader_source + assert "container_value: list = value" in loader_source + assert "kind == 5 or kind == 6" in loader_source + + frontend_source = frontend.read_text(encoding="utf-8") + assert "node.func.id == 'len'" in frontend_source + assert "self.constant(self.expression_kind(node.args[0]))" in frontend_source + assert "self.emit(Op.CALL, 2)" in frontend_source + assert frontend_source.count("self.emit(Op.CALL, 2)") == 1 + ast.parse(loader_source) + ast.parse(frontend_source) + + +def test_fails_closed_when_len_builtin_shape_changes( + tmp_path: Path, + monkeypatch, +) -> None: + loader = tmp_path / "loader.py" + frontend = tmp_path / "frontend.py" + loader.write_text( + LOADER_SOURCE.replace("return len(value)", "return 0"), + encoding="utf-8", + ) + frontend.write_text(FRONTEND_SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "LOADER_PATH", loader) + monkeypatch.setattr(normalizer, "FRONTEND_PATH", frontend) + + try: + normalizer.main() + except RuntimeError as error: + assert "unsafe object shape" in str(error) + else: + raise AssertionError("normalizer accepted an unexpected len builtin") diff --git a/tests/test_normalize_full_core_local_name_collisions.py b/tests/test_normalize_full_core_local_name_collisions.py new file mode 100644 index 00000000..48aa3bbf --- /dev/null +++ b/tests/test_normalize_full_core_local_name_collisions.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools.normalize_full_core_local_name_collisions import normalize_tree + + +def test_renames_nonparameter_locals_across_flattened_modules( + tmp_path: Path, +) -> None: + root = tmp_path / "portapy" + root.mkdir() + (root / "classes.py").write_text( + "class keyword:\n pass\n\nclass expr:\n pass\n", + encoding="utf-8", + ) + module = root / "runtime.py" + module.write_text( + '''def lower(items): + values = [] + for keyword in items: + expr = keyword.value + values.append(expr) + return values +''', + encoding="utf-8", + ) + + renamed, functions, files = normalize_tree(root) + + assert (renamed, functions, files) == (2, 1, 1) + source = module.read_text(encoding="utf-8") + assert "for __portapy_local_keyword in items:" in source + assert "__portapy_local_expr = __portapy_local_keyword.value" in source + assert "values.append(__portapy_local_expr)" in source + ast.parse(source) + + +def test_leaves_colliding_parameters_for_parameter_specific_passes( + tmp_path: Path, +) -> None: + root = tmp_path / "portapy" + root.mkdir() + (root / "module.py").write_text( + '''class keyword: + pass + +def lookup(keyword): + return keyword +''', + encoding="utf-8", + ) + + renamed, functions, files = normalize_tree(root) + + assert (renamed, functions, files) == (0, 0, 0) + source = (root / "module.py").read_text(encoding="utf-8") + assert "def lookup(keyword):" in source + assert "return keyword" in source + + +def test_processes_nested_scopes_independently(tmp_path: Path) -> None: + root = tmp_path / "portapy" + root.mkdir() + module = root / "module.py" + module.write_text( + '''class expr: + pass + +def outer(values): + expr = values[0] + def inner(values): + expr = values[1] + return expr + return expr, inner(values) +''', + encoding="utf-8", + ) + + renamed, functions, files = normalize_tree(root) + + assert (renamed, functions, files) == (2, 2, 1) + source = module.read_text(encoding="utf-8") + assert source.count("__portapy_local_expr") == 4 + ast.parse(source) diff --git a/tests/test_normalize_full_core_make_function.py b/tests/test_normalize_full_core_make_function.py new file mode 100644 index 00000000..85cefce9 --- /dev/null +++ b/tests/test_normalize_full_core_make_function.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from pathlib import Path + +from tools import normalize_full_core_make_function as normalizer + + +_ORIGINAL = ''' elif op is Op.MAKE_FUNCTION: + spec = frame.code.constants[instr.arg] + default_count = 0 + kw_default_count = 0 + annotations: dict[str, object] = {} + if isinstance(spec, tuple) and len(spec) == 4: + nested, default_count, kw_default_count, annotations = spec + elif isinstance(spec, tuple) and len(spec) == 2: + nested, default_count = spec + elif isinstance(spec, tuple) and len(spec) == 3: + nested, default_count, kw_default_count = spec + else: + nested = spec + if not isinstance(nested, CodeObject): _raise_typed("TypeError: invalid function constant") + count = default_count + kw_default_count + if len(frame.stack) < count: _raise_typed("RuntimeError: default stack underflow") + values = frame.stack[-count:] if count else [] + if count: del frame.stack[-count:] + defaults = values[:default_count] + kw_defaults = { + name: value for name, value in zip(nested.kwonly_names[-kw_default_count:], values[default_count:]) + } + closure = { + name: (frame.locals[name] if name in frame.locals else frame.closure[name]) + for name in nested.free_names + if name in frame.locals or (frame.closure is not None and name in frame.closure) + } + nested.validate() + function = Function(nested, frame.globals, defaults, kw_defaults, closure, self) + if annotations: + function._metadata["__annotations__"] = dict(annotations) + frame.stack.append(function) + elif op is Op.MAKE_CLASS: + pass +''' + +_NATIVE_SEMANTICS_INPUT = ''' elif op is Op.MAKE_FUNCTION: + spec = frame.code.constants[instr.arg] + default_count = 0 + kw_default_count = 0 + annotations: dict[str, object] = {} + nested = spec + if not isinstance(nested, CodeObject): _raise_typed("TypeError: invalid function constant") + count = default_count + kw_default_count + if len(frame.stack) < count: _raise_typed("RuntimeError: default stack underflow") + values = _full_core_probe_pop_tail(frame.stack, count) + defaults = _full_core_probe_copy_range(values, 0, default_count) + kw_defaults: dict[str, object] = {} + kw_name_start = len(nested.kwonly_names) - kw_default_count + kw_index = 0 + while kw_index < kw_default_count: + kw_defaults[nested.kwonly_names[kw_name_start + kw_index]] = values[default_count + kw_index] + kw_index += 1 + closure = { + name: (frame.locals[name] if name in frame.locals else frame.closure[name]) + for name in nested.free_names + if name in frame.locals or (frame.closure is not None and name in frame.closure) + } + nested.validate() + function = Function(nested, frame.globals, defaults, kw_defaults, closure, self) + frame.stack.append(function) + elif op is Op.MAKE_CLASS: + pass +''' + + +def test_unpacks_canonical_four_field_spec_without_introspection( + tmp_path: Path, monkeypatch, +) -> None: + path = tmp_path / "vm.py" + path.write_text(_ORIGINAL, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + source = path.read_text(encoding="utf-8") + assert "nested: CodeObject = spec[0]" in source + assert "default_count = spec[1]" in source + assert "kw_default_count = spec[2]" in source + assert "annotations: dict[str, object] = spec[3]" in source + assert "while default_index < default_count:" in source + assert "while kw_index < kw_default_count:" in source + assert "for name in nested.free_names:" in source + assert "len(spec)" not in source + assert "spec_size" not in source + assert "isinstance(spec, tuple)" not in source + assert "isinstance(nested, CodeObject)" not in source + assert "frame.stack[-count:]" not in source + assert "values[:default_count]" not in source + + +def test_accepts_native_semantics_intermediate_shape( + tmp_path: Path, monkeypatch, +) -> None: + path = tmp_path / "vm.py" + path.write_text(_NATIVE_SEMANTICS_INPUT, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + source = path.read_text(encoding="utf-8") + assert "nested: CodeObject = spec[0]" in source + assert "annotations: dict[str, object] = spec[3]" in source + assert "discarded_default = frame.stack.pop()" in source + assert "_full_core_probe_pop_tail(frame.stack, count)" not in source + assert "_full_core_probe_copy_range(values, 0, default_count)" not in source + + +def test_preserves_make_class_boundary(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "vm.py" + path.write_text(_ORIGINAL, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + source = path.read_text(encoding="utf-8") + assert source.count("elif op is Op.MAKE_FUNCTION:") == 1 + assert source.count("elif op is Op.MAKE_CLASS:") == 1 + + +def test_fails_closed_when_source_shape_changes(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "vm.py" + path.write_text( + ''' elif op is Op.MAKE_FUNCTION: + frame.stack.append(1) + elif op is Op.MAKE_CLASS: + pass +''', + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "source shape changed" in str(error) + else: + raise AssertionError("normalizer accepted an unknown MAKE_FUNCTION shape") diff --git a/tests/test_normalize_full_core_name_index.py b/tests/test_normalize_full_core_name_index.py new file mode 100644 index 00000000..183befec --- /dev/null +++ b/tests/test_normalize_full_core_name_index.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from pathlib import Path + +from tools import normalize_full_core_name_index as normalizer + + +SOURCE = '''class _Lowerer: + def name_index(self, value: str) -> int: + try: + return self.names.index(value) + except ValueError: + self.names.append(value) + return len(self.names) - 1 +''' + + +def test_replaces_exception_lookup_with_direct_loop( + tmp_path: Path, monkeypatch, +) -> None: + path = tmp_path / "frontend.py" + path.write_text(SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + source = path.read_text(encoding="utf-8") + assert "while index < len(self.names):" in source + assert "if self.names[index] == value:" in source + assert "self.names.append(value)" in source + assert ".names.index(" not in source + namespace: dict[str, object] = {} + exec(source, namespace) + lowerer = namespace["_Lowerer"]() + lowerer.names = ["alpha"] + assert lowerer.name_index("alpha") == 0 + assert lowerer.name_index("beta") == 1 + assert lowerer.names == ["alpha", "beta"] + + +def test_fails_closed_when_shape_changes(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "frontend.py" + path.write_text("class _Lowerer:\n pass\n", encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "expected one exception lookup" in str(error) + else: + raise AssertionError("normalizer accepted a missing name_index implementation") diff --git a/tests/test_normalize_full_core_native_argument_defaults.py b/tests/test_normalize_full_core_native_argument_defaults.py new file mode 100644 index 00000000..4b48362c --- /dev/null +++ b/tests/test_normalize_full_core_native_argument_defaults.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_core_native_argument_defaults as normalizer + + +_SOURCE = '''class AST: + pass + +class expr(AST): + pass + +class arg(AST): + def __init__(self, arg: str) -> None: + self.arg = arg + +class arguments(AST): + def __init__(self, posonlyargs: list[arg], args: list[arg], vararg: arg | None, + kwonlyargs: list[arg], kw_defaults: list[expr | None], + kwarg: arg | None, defaults: list[expr]) -> None: + self.posonlyargs = posonlyargs + self.args = args + self.vararg = vararg + self.kwonlyargs = kwonlyargs + self.kw_defaults = kw_defaults + self.kwarg = kwarg + self.defaults = defaults + +def _convert_arguments(node: A.FuncDef, lifted: dict[str, A.FuncDef]) -> arguments: + all_args = [arg(name) for name in node.params] + defaults: list[expr] = [] + first_default = len(node.defaults) + index = 0 + while index < len(node.defaults): + if node.defaults[index] is not None: + first_default = index + break + index += 1 + if first_default < len(node.defaults): + index = first_default + while index < len(node.defaults): + defaults.append(_convert_expr(node.defaults[index], lifted)) + index += 1 + return arguments([], all_args, None if node.vararg is None else arg(node.vararg), [], [], None if node.kwarg is None else arg(node.kwarg), defaults) +''' + + +def _module(path: Path) -> ast.Module: + return ast.parse(path.read_text(encoding="utf-8")) + + +def _function(path: Path) -> ast.FunctionDef: + functions = [ + node + for node in _module(path).body + if isinstance(node, ast.FunctionDef) and node.name == "_convert_arguments" + ] + assert len(functions) == 1 + return functions[0] + + +def _arguments_initializer(path: Path) -> ast.FunctionDef: + classes = [ + node + for node in _module(path).body + if isinstance(node, ast.ClassDef) and node.name == "arguments" + ] + assert len(classes) == 1 + initializers = [ + node + for node in classes[0].body + if isinstance(node, ast.FunctionDef) and node.name == "__init__" + ] + assert len(initializers) == 1 + return initializers[0] + + +def _isolate(path: Path, monkeypatch) -> None: + monkeypatch.setattr(normalizer, "PATH", path) + + +def test_pins_default_elements_and_storage_to_dict_backed_ast( + tmp_path: Path, monkeypatch, +) -> None: + path = tmp_path / "native_ast.py" + path.write_text(_SOURCE, encoding="utf-8") + _isolate(path, monkeypatch) + + assert normalizer.main() == 0 + + function = _function(path) + source = ast.unparse(function) + annotated_getattrs = [ + node + for node in ast.walk(function) + if isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == "native_defaults" + and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Name) + and node.value.func.id == "getattr" + and len(node.value.args) == 2 + and isinstance(node.value.args[1], ast.Constant) + and node.value.args[1].value == "defaults" + ] + assert len(annotated_getattrs) == 1 + assert "defaults: list[dict] = []" in source + assert source.count("default_node: dict = native_defaults[index]") == 2 + assert "converted_default: dict = _convert_expr(default_node, lifted)" in source + assert "defaults.append(converted_default)" in source + assert "_convert_expr(node.defaults[index], lifted)" not in source + + initializer = _arguments_initializer(path) + annotations = { + argument.arg: ast.unparse(argument.annotation) + for argument in initializer.args.args + if argument.annotation is not None + } + assert annotations["defaults"] == "list[dict]" + assert annotations["kw_defaults"] == "list[dict | None]" + + +def test_is_idempotent(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "native_ast.py" + path.write_text(_SOURCE, encoding="utf-8") + _isolate(path, monkeypatch) + + assert normalizer.main() == 0 + first = path.read_text(encoding="utf-8") + assert normalizer.main() == 0 + assert path.read_text(encoding="utf-8") == first + + +def test_fails_closed_for_unknown_shape(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "native_ast.py" + path.write_text("def _convert_arguments():\n pass\n", encoding="utf-8") + _isolate(path, monkeypatch) + + try: + normalizer.main() + except RuntimeError as error: + assert "native default" in str(error) + else: + raise AssertionError("normalizer accepted an unknown default bridge") diff --git a/tests/test_normalize_full_core_native_keyword_transport.py b/tests/test_normalize_full_core_native_keyword_transport.py new file mode 100644 index 00000000..b054d657 --- /dev/null +++ b/tests/test_normalize_full_core_native_keyword_transport.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_core_native_keyword_transport as normalizer + + +_SOURCE = '''def _raise_typed(message: str) -> None: + raise TypeError(message) + + +class SuperProxy: + pass + + +class VirtualMachine: + def _call(self, target: object, args: list[object], kwargs: dict[str, object] | None = None) -> object: + kwargs = kwargs or {} + return kwargs + + def _run_frame(self, frame: object) -> object: + target = object() + positional: list[object] = [] + names: list[object] = [] + values: list[object] = [] + if True: + if True: + kwargs: dict[str, object] = {} + for name, value in zip(names, values): + if name is None: + if not isinstance(value, dict): _raise_typed("TypeError: ** argument must be a mapping") + kwargs.update(value) + else: + kwargs[name] = value + if getattr(target, "__pyinbin_super__", False) and not positional and not kwargs: + instance = frame.locals.get("self") + cls = self._lexical_super_class(frame, instance) + frame.stack.append(SuperProxy(self, cls, instance)) + else: + frame.stack.append(self._call(target, positional, kwargs)) +''' + + +def _normalize(tmp_path: Path, monkeypatch) -> str: + path = tmp_path / "vm.py" + path.write_text(_SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "VM_PATH", path) + assert normalizer.main() == 0 + source = path.read_text(encoding="utf-8") + ast.parse(source) + return source + + +def _vm(source: str) -> object: + namespace: dict[str, object] = {} + exec(source, namespace) + return namespace["VirtualMachine"]() + + +def _expect_type_error(callback, message: str) -> None: + try: + callback() + except TypeError as error: + assert message in str(error) + else: + raise AssertionError("expected TypeError") + + +def test_transports_typed_keyword_names_and_values( + tmp_path: Path, + monkeypatch, +) -> None: + source = _normalize(tmp_path, monkeypatch) + + assert "keyword_names: list[str] | None = None" in source + assert "keyword_values: list[object] | None = None" in source + assert "transported_kwargs: dict[str, object] = {}" in source + assert "keyword_name: str = keyword_names[keyword_index]" in source + assert "mapping_name: str = raw_mapping_name" in source + assert "kwargs = transported_kwargs" in source + assert "keyword_names: list[str] = []" in source + assert 'keyword_names.append("")' in source + assert "keyword_name: str = name" in source + assert "self._call(target, positional, None, keyword_names, values)" in source + assert "keyword_names: list[object]" not in source + assert "for name, value in zip(names, values):" not in source + + vm = _vm(source) + result = vm._call( + object(), + [], + None, + ["direct", ""], + [1, {"mapped": 2, "": 3}], + ) + assert result == {"direct": 1, "mapped": 2, "": 3} + + +def test_rejects_invalid_or_duplicate_mapping_keywords( + tmp_path: Path, + monkeypatch, +) -> None: + vm = _vm(_normalize(tmp_path, monkeypatch)) + + _expect_type_error( + lambda: vm._call(object(), [], None, [""], [{1: "bad"}]), + "keywords must be strings", + ) + _expect_type_error( + lambda: vm._call( + object(), + [], + None, + ["direct", ""], + [1, {"direct": 2}], + ), + "multiple values for keyword argument", + ) + + +def test_empty_keyword_unpack_preserves_lexical_super( + tmp_path: Path, + monkeypatch, +) -> None: + source = _normalize(tmp_path, monkeypatch) + + assert "has_effective_keywords = False" in source + assert "if len(value) > 0:" in source + assert "and not has_effective_keywords" in source + assert "and not keyword_names" not in source + + +def test_fails_closed_when_call_shape_changes( + tmp_path: Path, + monkeypatch, +) -> None: + path = tmp_path / "vm.py" + path.write_text("class VirtualMachine:\n pass\n", encoding="utf-8") + monkeypatch.setattr(normalizer, "VM_PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "native keyword receiver" in str(error) + else: + raise AssertionError("normalizer accepted an unknown VM call shape") + + +def test_keyword_transport_runs_before_every_other_normalizer() -> None: + validation_path = Path(__file__).parents[1] / "tools" / "normalize_full_core_validation.py" + module = ast.parse(validation_path.read_text(encoding="utf-8")) + main = next( + node + for node in module.body + if isinstance(node, ast.FunctionDef) and node.name == "main" + ) + assignment = next( + node + for node in main.body + if isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == "steps" + ) + assert isinstance(assignment.value, ast.Tuple) + names = [ + item.elts[0].value + for item in assignment.value.elts + if isinstance(item, ast.Tuple) + and len(item.elts) == 2 + and isinstance(item.elts[0], ast.Constant) + ] + assert names[0] == "native_keyword_transport" diff --git a/tests/test_normalize_full_core_native_node_fields.py b/tests/test_normalize_full_core_native_node_fields.py new file mode 100644 index 00000000..7ca60569 --- /dev/null +++ b/tests/test_normalize_full_core_native_node_fields.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import ast + +from tools import normalize_full_core_native_node_fields as normalizer + + +def test_opaque_node_fields_use_runtime_getattr() -> None: + module = ast.parse( + ''' +def _convert_stmt(node: object, lifted: dict, other: object): + if isinstance(node, ExprStmt): + return Expr(_convert_expr(node.expr, lifted)) + return (node.name, other.value) +''' + ) + function = module.body[0] + assert isinstance(function, ast.FunctionDef) + + count, fields = normalizer._normalize_function(function) + ast.fix_missing_locations(module) + source = ast.unparse(module) + + assert count == 2 + assert fields == {"expr", "name"} + assert "node.expr" not in source + assert "node.name" not in source + assert "getattr(node, 'expr')" in source + assert "getattr(node, 'name')" in source + assert "other.value" in source diff --git a/tests/test_normalize_full_core_native_parser_expressions.py b/tests/test_normalize_full_core_native_parser_expressions.py new file mode 100644 index 00000000..dcfcfcfd --- /dev/null +++ b/tests/test_normalize_full_core_native_parser_expressions.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_core_native_parser_expressions as normalizer + + +SOURCE = '''class _npr_ast_nodes_ExprStmt: + expr: "Expr" + pos: object + +class _npr_parser_Parser: + def _parse_stmt(self): + pos = self._peek().pos + expr = self._parse_expr() + if isinstance(expr, Name): + return expr + if condition: + value = self._parse_expr() + other = self._peek() + return expr + + def another_method(self): + untouched = self._parse_expr() + return untouched +''' + + +def test_boxes_results_field_and_fast_paths_expression_statements( + tmp_path: Path, monkeypatch +) -> None: + path = tmp_path / "native_ast.py" + path.write_text(SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + source = path.read_text(encoding="utf-8") + assert "class _npr_ast_nodes_ExprStmt:" in source + assert "expr: dict" in source + assert ( + "__pyinbin_native_expr_values: list[dict] = [self._parse_expr()]" + in source + ) + assert "return _npr_ast_nodes_ExprStmt(__pyinbin_native_expr_values[0], pos)" in source + assert "expr: dict = __pyinbin_native_expr_values[0]" in source + assert "value: dict = self._parse_expr()" in source + assert "untouched = self._parse_expr()" in source + assert "other = self._peek()" in source + assert "if self._check('NEWLINE'):" in source + assert "self._eat()" in source + assert source.index("if self._check('NEWLINE'):") < source.index( + "if isinstance(expr, Name):" + ) + ast.parse(source) + + +def test_fails_closed_without_parser_method(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "native_ast.py" + path.write_text( + 'class _npr_ast_nodes_ExprStmt:\n expr: "Expr"\n', + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "Parser class is missing" in str(error) + else: + raise AssertionError("normalizer accepted a missing embedded parser") + + +def test_fails_closed_without_expr_stmt_field(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "native_ast.py" + path.write_text( + SOURCE.replace(' expr: "Expr"\n', " value: object\n"), + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "ExprStmt.expr expected one field" in str(error) + else: + raise AssertionError("normalizer accepted a missing ExprStmt.expr field") + + +def test_fails_closed_without_main_expr_assignment( + tmp_path: Path, monkeypatch +) -> None: + path = tmp_path / "native_ast.py" + path.write_text( + SOURCE.replace(" expr = self._parse_expr()\n", " expr = 1\n"), + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "fast path expected one insertion" in str(error) + else: + raise AssertionError("normalizer accepted a missing expression fast path") diff --git a/tests/test_normalize_full_core_native_parser_target_dispatch.py b/tests/test_normalize_full_core_native_parser_target_dispatch.py new file mode 100644 index 00000000..2ddeaf3c --- /dev/null +++ b/tests/test_normalize_full_core_native_parser_target_dispatch.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from pathlib import Path + +from tools import normalize_full_core_native_parser_target_dispatch as normalizer + + +def test_parser_target_dispatch_keeps_expression_in_runtime_box( + tmp_path: Path, + monkeypatch, +) -> None: + target = tmp_path / "native_ast.py" + target.write_text( + ''' +class _npr_parser_Parser: + def _parse_stmt(self): + __pyinbin_native_expr_values: list[dict] = [self._parse_expr()] + if self._check("NEWLINE"): + return ExprStmt(__pyinbin_native_expr_values[0]) + expr: dict = __pyinbin_native_expr_values[0] + if isinstance(expr, Name): + return expr + if isinstance(expr, Subscript): + return Assign(expr, expr) + if isinstance(expr, _npr_ast_nodes_Attr): + return AttrAssign(obj=expr.obj, name=expr.name) + return ExprStmt(expr) +''', + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", target) + + assert normalizer.main() == 0 + + source = target.read_text(encoding="utf-8") + assert "expr: dict" not in source + assert "isinstance(expr" not in source + assert "return expr" not in source + assert source.count("__pyinbin_native_expr_values[0]") >= 8 + assert ( + "_native_attr_obj_values_0: list[dict] = " + "[getattr(__pyinbin_native_expr_values[0], 'obj')]" + ) in source + assert ( + "_native_attr_name_values_0: list[str] = " + "[getattr(__pyinbin_native_expr_values[0], 'name')]" + ) in source + assert "__pyinbin_native_expr_values[0].obj" not in source + assert "__pyinbin_native_expr_values[0].name" not in source + assert "obj=_native_attr_obj_values_0[0]" in source + assert "name=_native_attr_name_values_0[0]" in source diff --git a/tests/test_normalize_full_core_native_statement_bodies.py b/tests/test_normalize_full_core_native_statement_bodies.py new file mode 100644 index 00000000..cca1f47c --- /dev/null +++ b/tests/test_normalize_full_core_native_statement_bodies.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import ast + +from tools import normalize_full_core_native_statement_bodies as normalizer + + +def test_control_flow_bodies_are_loaded_and_converted_before_construction() -> None: + module = ast.parse( + ''' +def _convert_stmt(node: object, lifted: dict): + if isinstance(node, _npr_ast_nodes_For): + return For(target, iterator, _convert_body(node.body, lifted), _convert_body(node.orelse, lifted)) + return None +''' + ) + function = normalizer._convert_stmt(module) + + assert normalizer._normalize_function(function) == 2 + + ast.fix_missing_locations(module) + source = ast.unparse(module) + assert "_native_for_body_body: list[object] = getattr(node, 'body')" in source + assert "_native_for_orelse_body: list[object] = getattr(node, 'orelse')" in source + assert "_native_converted_for_body_body: list[stmt] = _convert_body(" in source + assert "_native_converted_for_orelse_body: list[stmt] = _convert_body(" in source + assert "_convert_body(node.body" not in source + assert "_convert_body(node.orelse" not in source + assert "For(target, iterator, _native_converted_for_body_body, _native_converted_for_orelse_body)" in source diff --git a/tests/test_normalize_full_core_parameter_markers.py b/tests/test_normalize_full_core_parameter_markers.py new file mode 100644 index 00000000..b78d1ac3 --- /dev/null +++ b/tests/test_normalize_full_core_parameter_markers.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_core_parameter_markers as normalizer + + +SOURCE = '''class _npr_parser_Parser: + def _parse_funcdef(self, decorators=None): + start = self._peek().pos + self._expect("KEYWORD", "def") + name = self._expect("NAME").value + self._expect("OP", "(") + params = [] + defaults = [] + param_types = [] + vararg = None + kwarg = None + first = True + while not self._check("OP", ")"): + if not first: + self._expect("OP", ",") + first = False + self._parse_param(params, defaults, param_types) + self._expect("OP", ")") + self._expect("OP", ":") + body = self._parse_block() + return _npr_ast_nodes_FuncDef(name=name, params=params, body=body) + +def _convert_arguments(node, lifted): + all_args = [AstArg(name) for name in node.params] + defaults = [] + return arguments([], all_args, None if node.vararg is None else AstArg(node.vararg), [], [], None if node.kwarg is None else AstArg(node.kwarg), defaults) +''' + + +def test_preserves_slash_and_star_partitions( + tmp_path: Path, + monkeypatch, +) -> None: + path = tmp_path / "native_ast.py" + path.write_text(SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + source = path.read_text(encoding="utf-8") + assert "self._check('OP', '/')" in source + assert "__portapy_posonly_marker__" in source + assert "__portapy_kwonly_marker__" in source + assert "positional_only.append(parameter)" in source + assert "keyword_only.append(parameter)" in source + assert "keyword_defaults.append(None)" in source + assert "return arguments(positional_only, regular, vararg_node" in source + assert "return arguments([], all_args" not in source + ast.parse(source) + + +def test_fails_closed_for_unknown_bridge_shape( + tmp_path: Path, + monkeypatch, +) -> None: + path = tmp_path / "native_ast.py" + path.write_text( + SOURCE.replace( + "return arguments([], all_args, None if node.vararg is None else AstArg(node.vararg), [], [], None if node.kwarg is None else AstArg(node.kwarg), defaults)", + "return arguments([], all_args, None, [], [], None, defaults)", + ), + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "flattened source shape" in str(error) + else: + raise AssertionError("normalizer accepted an unexpected argument bridge") diff --git a/tests/test_normalize_full_core_parameter_name_collisions.py b/tests/test_normalize_full_core_parameter_name_collisions.py new file mode 100644 index 00000000..15331664 --- /dev/null +++ b/tests/test_normalize_full_core_parameter_name_collisions.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools.normalize_full_core_parameter_name_collisions import normalize_tree + + +def test_repairs_explicit_and_generated_parameter_collisions( + tmp_path: Path, +) -> None: + root = tmp_path / "portapy" + root.mkdir() + module = root / "module.py" + module.write_text( + '''from dataclasses import dataclass, field + +class alias: + pass + +class arguments: + pass + +@dataclass +class ImportNode: + module: str + alias: str | None = None + pos: object = field(default_factory=lambda: NO_POS) + +def dispatch(arguments): + return arguments + +def build(value): + return ImportNode(module='demo', alias=value) +''', + encoding="utf-8", + ) + + explicit, generated, classes, calls = normalize_tree(root) + + assert (explicit, generated, classes, calls) == (1, 1, 1, 1) + source = module.read_text(encoding="utf-8") + assert "def dispatch(__portapy_param_arguments):" in source + assert "return __portapy_param_arguments" in source + assert "def __init__(self, module: str, __portapy_param_alias: str | None=None" in source + assert "self.alias = __portapy_param_alias" in source + assert "pos: object=NO_POS" in source + assert "ImportNode(module='demo', __portapy_param_alias=value)" in source + ast.parse(source) + + +def test_preserves_positional_calls_and_noncolliding_parameters( + tmp_path: Path, +) -> None: + root = tmp_path / "portapy" + root.mkdir() + module = root / "module.py" + module.write_text( + '''class keyword: + pass + +def lookup(keyword, fallback): + return keyword or fallback + +def run(value): + return lookup(value, None) +''', + encoding="utf-8", + ) + + explicit, generated, classes, calls = normalize_tree(root) + + assert (explicit, generated, classes, calls) == (1, 0, 0, 0) + source = module.read_text(encoding="utf-8") + assert "def lookup(__portapy_param_keyword, fallback):" in source + assert "return __portapy_param_keyword or fallback" in source + assert "lookup(value, None)" in source + + +def test_rejects_mutable_default_factories_for_colliding_fields( + tmp_path: Path, +) -> None: + root = tmp_path / "portapy" + root.mkdir() + (root / "module.py").write_text( + '''from dataclasses import dataclass, field + +class values: + pass + +@dataclass +class Node: + values: list = field(default_factory=list) +''', + encoding="utf-8", + ) + + try: + normalize_tree(root) + except RuntimeError as error: + assert "unsupported non-lambda default factory" in str(error) + else: + raise AssertionError("mutable generated default was accepted") diff --git a/tests/test_normalize_full_core_parser_errors.py b/tests/test_normalize_full_core_parser_errors.py new file mode 100644 index 00000000..3ab3915e --- /dev/null +++ b/tests/test_normalize_full_core_parser_errors.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_core_parser_errors as normalizer + + +SOURCE = '''class _npr_parser_Parser: + def _parse_primary(self): + t = self._peek() + if t.kind == "INT": + return t + raise _npr_errors_ParseError( + f"unexpected token {t.kind} {t.value!r}", + t.pos, + ErrorCode.P_UNEXPECTED_TOKEN, + ) +''' + + +def test_replaces_dynamic_unexpected_token_message( + tmp_path: Path, monkeypatch +) -> None: + path = tmp_path / "native_ast.py" + path.write_text(SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + source = path.read_text(encoding="utf-8") + assert "raise _npr_errors_ParseError('unexpected token'," in source + assert "t.pos" in source + assert "ErrorCode.P_UNEXPECTED_TOKEN" in source + assert "t.value" not in source + ast.parse(source) + + +def test_fails_closed_without_target_message(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "native_ast.py" + path.write_text( + SOURCE.replace("unexpected token", "different diagnostic"), + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "expected one rewrite" in str(error) + else: + raise AssertionError("normalizer accepted a missing target diagnostic") diff --git a/tests/test_normalize_full_core_pattern_constructor_collisions.py b/tests/test_normalize_full_core_pattern_constructor_collisions.py new file mode 100644 index 00000000..e2f59b7e --- /dev/null +++ b/tests/test_normalize_full_core_pattern_constructor_collisions.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_core_pattern_constructor_collisions as normalizer + + +SOURCE = '''class pattern: + pass + +class MatchAs: + def __init__(self, pattern: pattern | None = None, name: str | None = None): + self.pattern = pattern + self.name = name + +class match_case: + def __init__(self, pattern: pattern, guard: object, body: list): + self.pattern = pattern + self.guard = guard + self.body = body + +def build(value): + first = MatchAs(pattern=value, name='captured') + second = match_case(pattern=value, guard=None, body=[]) + return first, second +''' + + +def test_repairs_pattern_parameters_and_keyword_calls( + tmp_path: Path, + monkeypatch, +) -> None: + path = tmp_path / "native_ast.py" + path.write_text(SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + source = path.read_text(encoding="utf-8") + assert "def __init__(self, pattern_value: pattern | None=None" in source + assert "def __init__(self, pattern_value: pattern, guard: object" in source + assert source.count("self.pattern = pattern_value") == 2 + assert "MatchAs(pattern_value=value, name='captured')" in source + assert "match_case(pattern_value=value, guard=None, body=[])" in source + ast.parse(source) + + +def test_positional_calls_remain_unchanged(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "native_ast.py" + path.write_text( + SOURCE.replace( + "MatchAs(pattern=value, name='captured')", + "MatchAs(value, 'captured')", + ).replace( + "match_case(pattern=value, guard=None, body=[])", + "match_case(value, None, [])", + ), + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + source = path.read_text(encoding="utf-8") + assert "MatchAs(value, 'captured')" in source + assert "match_case(value, None, [])" in source + + +def test_fails_closed_when_pattern_class_is_missing( + tmp_path: Path, + monkeypatch, +) -> None: + path = tmp_path / "native_ast.py" + path.write_text(SOURCE.replace("class match_case:", "class missing_case:"), encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "missing" in str(error) + else: + raise AssertionError("normalizer accepted a missing pattern constructor") diff --git a/tests/test_normalize_full_core_pop_top.py b/tests/test_normalize_full_core_pop_top.py new file mode 100644 index 00000000..a5564363 --- /dev/null +++ b/tests/test_normalize_full_core_pop_top.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_core_pop_top as normalizer + + +FRONTEND_SOURCE = '''class _Lowerer: + def first(self): + self.emit(Op.POP_TOP) + + def second(self, condition): + if condition: + self.emit(Op.POP_TOP) + + def third(self, condition): + if condition: + if condition: + self.emit(Op.POP_TOP) + + def fourth(self): + self.emit(Op.POP_TOP) + + def fifth(self): + self.emit(Op.POP_TOP) +''' + + +def test_inlines_all_discard_emissions_with_original_indentation( + tmp_path: Path, monkeypatch +) -> None: + path = tmp_path / "frontend.py" + path.write_text(FRONTEND_SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + source = path.read_text(encoding="utf-8") + assert "self.emit(Op.POP_TOP)" not in source + assert "def discard_top" not in source + assert source.count('self.name_index("__pyinbin_internal_discard")') == 10 + assert source.count("Op.STORE_NAME,") == 5 + assert source.count("Op.DELETE_NAME,") == 5 + ast.parse(source) + + +def test_fails_closed_when_emission_count_changes( + tmp_path: Path, monkeypatch +) -> None: + path = tmp_path / "frontend.py" + path.write_text( + FRONTEND_SOURCE.replace(" self.emit(Op.POP_TOP)\n", "", 1), + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "expected 5 emissions" in str(error) + else: + raise AssertionError("normalizer accepted an unexpected POP_TOP count") diff --git a/tests/test_normalize_full_core_runtime_dispatch.py b/tests/test_normalize_full_core_runtime_dispatch.py new file mode 100644 index 00000000..5592d31b --- /dev/null +++ b/tests/test_normalize_full_core_runtime_dispatch.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_core_runtime_dispatch as normalizer + + +SOURCE = '''class VirtualMachine: + def bind(self, target, name): + checks = [ + name in target.code.posonly_names, + name in target.code.arg_names, + name in target.code.kwonly_names, + name not in target.code.posonly_names, + name not in target.code.arg_names, + name not in target.code.kwonly_names, + ] + return checks + + def run(self, op, frame, instr): + if op is Op.GET_ITER: + value = frame.stack.pop() + if isinstance(value, dict) or type(value).__name__ in {"dict_keys"}: + value = list(value) + frame.stack.append(iter(value)) + elif op is Op.MAKE_CLASS: + class_keywords = frame.stack.pop() if has_keywords else {} + if not isinstance(class_keywords, dict): + _raise_typed("TypeError: class keyword arguments must be a dict") + frame.stack.append(class_keywords) + elif op is Op.IMPORT_NAME: + loader = frame.globals.get("__pyinbin_import__") + if not callable(loader): + _raise_typed("ImportError: loader is not configured") + imported = frame.code.names[instr.arg] + top_level = imported.split(".", 1)[0] + loader(top_level) + frame.stack.append(loader(imported)) + elif op is Op.IMPORT_FROM: + module = frame.stack.pop() + member = frame.code.names[instr.arg] + loader = frame.globals.get("__pyinbin_import__") + module_name = getattr(module, "__name__", None) + if not callable(loader) or not isinstance(module_name, str): + raise AttributeError(member) + value = loader(module_name) + frame.stack.append(value) + elif op is Op.IMPORT_ROOT: + loader = frame.globals.get("__pyinbin_import__") + if not callable(loader): + _raise_typed("ImportError: loader is not configured") + imported = frame.code.names[instr.arg] + top_level = imported.split(".", 1)[0] + loader(top_level) + loader(imported) + loader("sys") + frame.stack.append(loader(top_level)) + elif op is Op.IMPORT_RELATIVE_FROM: + loader = frame.globals.get("__pyinbin_import__") + if not callable(loader): + _raise_typed("ImportError: loader is not configured") + base = "pkg" + member = "item" + loader(base) + loader(base + "." + member) + loader(base) + loader(base + "." + member) +''' + + +def test_removes_native_runtime_dispatch_hazards( + tmp_path: Path, + monkeypatch, +) -> None: + path = tmp_path / "vm.py" + path.write_text(SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + source = path.read_text(encoding="utf-8") + assert "def _full_core_native_name_in" in source + assert "frame.stack.append(iter(value))" in source + assert "type(value).__name__" not in source + assert "callable(loader)" not in source + assert "loader is None" in source + assert "loader(imported)" in source + assert "loader(top_level)" in source + assert "self._call(loader" not in source + assert "class keyword arguments must be a dict" not in source + assert "_full_core_native_name_in(target.code.arg_names, name)" in source + assert "name in target.code.arg_names" not in source + ast.parse(source) + + +def test_fails_closed_without_all_import_branches( + tmp_path: Path, + monkeypatch, +) -> None: + path = tmp_path / "vm.py" + path.write_text( + SOURCE.replace("elif op is Op.IMPORT_RELATIVE_FROM:", "elif op is Op.OTHER:"), + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "required branches" in str(error) + else: + raise AssertionError("normalizer accepted a missing import branch") diff --git a/tests/test_normalize_full_core_runtime_execution.py b/tests/test_normalize_full_core_runtime_execution.py new file mode 100644 index 00000000..aed5e097 --- /dev/null +++ b/tests/test_normalize_full_core_runtime_execution.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_core_runtime_execution as normalizer + + +SOURCE = '''class VirtualMachine: + def __init__(self): + self.value = 0 + + def _lookup(self, frame, name): + if name in frame.locals: + return frame.locals[name] + _raise_typed(f"NameError: name {name!r} is not defined") + + def _exception_matches(self, value, expected): + return False + + def _run_frame(self, frame): + while True: + try: + op = current_op + if op is Op.GET_ITER: + value = frame.stack.pop() + frame.stack.append(iter(value)) + elif op is Op.FOR_ITER: + if not frame.stack: + frame.ip = instr.arg + continue + try: + frame.stack.append(next(frame.stack[-1])) + except StopIteration: + frame.stack.pop() + frame.ip = instr.arg + elif op is Op.MAKE_FUNCTION: + closure = {} + for name in nested.free_names: + if name in frame.locals: + closure[name] = frame.locals[name] + elif frame.closure is not None and name in frame.closure: + closure[name] = frame.closure[name] + except BaseException as exc: + frame.stack.append(exc) +''' + + +def test_installs_explicit_native_execution_state( + tmp_path: Path, + monkeypatch, +) -> None: + path = tmp_path / "vm.py" + path.write_text(SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + source = path.read_text(encoding="utf-8") + assert "class _NativeSequenceIterator:" in source + assert "class _NativeCaughtException:" in source + assert "frame.stack.append(_NativeSequenceIterator(sequence))" in source + assert "iterator: _NativeSequenceIterator = frame.stack[-1]" in source + assert "value = iterator.values[iterator.index]" in source + assert "while closure_index < len(nested.free_names):" in source + assert "exc = _NativeCaughtException(exc)" in source + assert "if isinstance(value, _NativeCaughtException):" in source + assert "self._native_error_kind = 'NameError'" in source + assert "self._native_error_kind = ''" in source + assert "frame.stack.append(iter(value))" not in source + assert "next(frame.stack[-1])" not in source + assert "for name in nested.free_names" not in source + ast.parse(source) + + +def test_fails_closed_without_closure_loop( + tmp_path: Path, + monkeypatch, +) -> None: + path = tmp_path / "vm.py" + path.write_text( + SOURCE.replace("for name in nested.free_names:", "for name in []:"), + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "closure loop" in str(error) + else: + raise AssertionError("normalizer accepted a missing closure loop") diff --git a/tests/test_normalize_full_core_runtime_specs.py b/tests/test_normalize_full_core_runtime_specs.py new file mode 100644 index 00000000..556cedb3 --- /dev/null +++ b/tests/test_normalize_full_core_runtime_specs.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_core_runtime_specs as normalizer + + +BYTECODE = '''from __future__ import annotations + +class CodeObject: + pass +''' + +FRONTEND = '''from .bytecode import CodeObject, Instruction, Op + +class Lowerer: + def lower(self, node, body, base_count, has_keywords, arg_specs): + keyword_names: list[object] = [] + names = tuple(keyword_names) + self.emit(Op.CALL_KW, self.constant((tuple(arg_specs), names))) + spec = (node.name, body.finish(), base_count, has_keywords) + self.emit(Op.MAKE_CLASS, self.constant(spec)) +''' + +VM = '''from .bytecode import CodeObject, Op + +class VirtualMachine: + def run(self, op, frame, instr): + if op is Op.MAKE_CLASS: + spec = frame.code.constants[instr.arg] + if not isinstance(spec, tuple) or len(spec) not in (3, 4): + _raise_typed("TypeError: invalid class constant") + class_name, body, base_count = spec[:3] + has_keywords = bool(spec[3]) if len(spec) == 4 else False + if not isinstance(class_name, str) or not isinstance(body, CodeObject): + _raise_typed("TypeError: invalid class constant") + bases = _full_core_probe_pop_tail(frame.stack, base_count) + elif op is Op.CALL_KW: + spec = frame.code.constants[instr.arg] + if not isinstance(spec, tuple) or len(spec) != 2: + _raise_typed("RuntimeError: invalid keyword call") + positional_spec, names = spec + if isinstance(positional_spec, int): + positional_spec = tuple(False for _ in range(positional_spec)) + if not isinstance(positional_spec, tuple): + _raise_typed("RuntimeError: invalid positional call") + positional_count = len(positional_spec) +''' + + +def test_converts_opcode_specs_to_typed_objects( + tmp_path: Path, + monkeypatch, +) -> None: + bytecode = tmp_path / "bytecode.py" + frontend = tmp_path / "frontend.py" + vm = tmp_path / "vm.py" + bytecode.write_text(BYTECODE, encoding="utf-8") + frontend.write_text(FRONTEND, encoding="utf-8") + vm.write_text(VM, encoding="utf-8") + monkeypatch.setattr(normalizer, "BYTECODE_PATH", bytecode) + monkeypatch.setattr(normalizer, "FRONTEND_PATH", frontend) + monkeypatch.setattr(normalizer, "VM_PATH", vm) + + assert normalizer.main() == 0 + + bytecode_source = bytecode.read_text(encoding="utf-8") + frontend_source = frontend.read_text(encoding="utf-8") + vm_source = vm.read_text(encoding="utf-8") + assert "class _NativeKeywordCallSpec:" in bytecode_source + assert "class _NativeClassSpec:" in bytecode_source + assert "self.positional_spec = positional_spec" in bytecode_source + assert "self.body = body" in bytecode_source + assert "_NativeKeywordCallSpec(arg_specs, names)" in frontend_source + assert ( + "_NativeClassSpec(node.name, body.finish(), base_count, has_keywords)" + in frontend_source + ) + assert "spec: _NativeKeywordCallSpec" in vm_source + assert "positional_spec = spec.positional_spec" in vm_source + assert "names = spec.names" in vm_source + assert "spec: _NativeClassSpec" in vm_source + assert "class_name = spec.class_name" in vm_source + assert "body = spec.body" in vm_source + assert "len(spec)" not in vm_source + assert "isinstance(spec, tuple)" not in vm_source + ast.parse(bytecode_source) + ast.parse(frontend_source) + ast.parse(vm_source) + + +def test_fails_closed_when_frontend_shape_changes( + tmp_path: Path, + monkeypatch, +) -> None: + bytecode = tmp_path / "bytecode.py" + frontend = tmp_path / "frontend.py" + vm = tmp_path / "vm.py" + bytecode.write_text(BYTECODE, encoding="utf-8") + frontend.write_text( + FRONTEND.replace("names = tuple(keyword_names)", "names = keyword_names"), + encoding="utf-8", + ) + vm.write_text(VM, encoding="utf-8") + monkeypatch.setattr(normalizer, "BYTECODE_PATH", bytecode) + monkeypatch.setattr(normalizer, "FRONTEND_PATH", frontend) + monkeypatch.setattr(normalizer, "VM_PATH", vm) + + try: + normalizer.main() + except RuntimeError as error: + assert "frontend runtime specs" in str(error) + else: + raise AssertionError("normalizer accepted an unknown frontend shape") diff --git a/tests/test_normalize_full_core_string_addition.py b/tests/test_normalize_full_core_string_addition.py new file mode 100644 index 00000000..28f88a9d --- /dev/null +++ b/tests/test_normalize_full_core_string_addition.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +from tools import normalize_full_core_string_addition as normalizer + + +NATIVE_AST_SOURCE = ''' +class A: + class BinOp: pass +class expr: pass +class BinOp(expr): + def __init__(self, left, op, right): + self.left = left + self.op = op + self.right = right +_BIN_OPS = {"+": object()} +def _convert_expr(node, lifted): + if isinstance(node, A.BinOp): + return BinOp(_convert_expr(node.left, lifted), _BIN_OPS[node.op], _convert_expr(node.right, lifted)) + return expr() +''' + +FRONTEND_SOURCE = ''' +class Lowerer: + def expr(self, node): + if False: + pass + elif isinstance(node, ast.BinOp) and _binary_opcode(node.op) is not None: + self.expr(node.left) + self.expr(node.right) + self.emit(_binary_opcode(node.op)) +''' + +VM_SOURCE = ''' +class VirtualMachine: + def run(self, frame, instr, op): + left = None + right = None + if op is Op.BINARY_ADD: frame.stack.append(left + right) +''' + + +def test_installs_typed_native_string_addition( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + native_ast = tmp_path / "native_ast.py" + frontend = tmp_path / "frontend.py" + vm = tmp_path / "vm.py" + native_ast.write_text(NATIVE_AST_SOURCE, encoding="utf-8") + frontend.write_text(FRONTEND_SOURCE, encoding="utf-8") + vm.write_text(VM_SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "NATIVE_AST_PATH", native_ast) + monkeypatch.setattr(normalizer, "FRONTEND_PATH", frontend) + monkeypatch.setattr(normalizer, "VM_PATH", vm) + + assert normalizer.main() == 0 + + native_text = native_ast.read_text(encoding="utf-8") + assert "converted._native_kind = 4" in native_text + assert "node.op == '+'" in native_text + + frontend_text = frontend.read_text(encoding="utf-8") + assert "self.emit(binary_opcode, binary_kind)" in frontend_text + assert "binary_kind = _TRUTH_STRING" in frontend_text + assert "binary_kind = 8" in frontend_text + + vm_text = vm.read_text(encoding="utf-8") + assert "def _full_core_probe_concat_strings" in vm_text + assert 'return f"{left}{right}"' in vm_text + assert "instr.arg == 4" in vm_text + assert "instr.arg == 8" in vm_text + assert "can only concatenate string to string" in vm_text + + ast.parse(native_text) + ast.parse(frontend_text) + ast.parse(vm_text) diff --git a/tests/test_normalize_full_core_tracebacks.py b/tests/test_normalize_full_core_tracebacks.py new file mode 100644 index 00000000..a30167e8 --- /dev/null +++ b/tests/test_normalize_full_core_tracebacks.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tools import normalize_full_core_tracebacks as normalizer + + +_SOURCE = '''class VirtualMachine: + def __init__(self) -> None: + self._synthetic_tracebacks: dict[int, "_PyTBProxy"] = {} + + def read(self, target): + value = self._synthetic_tracebacks.get(id(target), target.__traceback__) + return value + + def write(self, frame, exc): + if True: + try: + pass + except BaseException as exc: + if isinstance(exc, BaseException) and not isinstance(exc, PyException): + tb_frame = _PyTBFrameProxy(frame.code, frame.globals, None) + prior = self._synthetic_tracebacks.get(id(exc)) + self._synthetic_tracebacks[id(exc)] = _PyTBProxy(tb_frame, prior) + return exc +''' + + +def test_disables_host_style_native_traceback_storage( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "vm.py" + path.write_text(_SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + result = path.read_text(encoding="utf-8") + assert "_synthetic_tracebacks" not in result + assert "value = None" in result + assert "_PyTBProxy(tb_frame, prior)" not in result + assert "return exc" in result + + +def test_rejects_missing_traceback_storage_shape( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "vm.py" + path.write_text("class VirtualMachine: pass\n", encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + with pytest.raises(RuntimeError, match="traceback table annotation"): + normalizer.main() diff --git a/tests/test_normalize_full_core_validation_order.py b/tests/test_normalize_full_core_validation_order.py new file mode 100644 index 00000000..bf9ad8fc --- /dev/null +++ b/tests/test_normalize_full_core_validation_order.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from pathlib import Path + + +VALIDATION = ( + Path(__file__).parents[1] / "tools" / "normalize_full_core_validation.py" +) + + +def test_match_defaults_precede_constructor_collision_renames() -> None: + source = VALIDATION.read_text(encoding="utf-8") + + extended = source.index('("extended_semantics"') + expr_stmt = source.index('("expr_stmt_initializer"') + pattern_collision = source.index('("pattern_constructor_collisions"') + node_fields = source.index('("native_node_fields"') + + assert extended < expr_stmt < pattern_collision < node_fields + + +def test_typed_container_access_follows_reference_data_access() -> None: + source = VALIDATION.read_text(encoding="utf-8") + + data_access = source.index('("reference_data_access"') + container_access = source.index('("reference_container_access"') + handle_kind_access = source.index('("reference_handle_kind_access"') + + assert data_access < container_access < handle_kind_access diff --git a/tests/test_normalize_full_reference_bytes_literals.py b/tests/test_normalize_full_reference_bytes_literals.py new file mode 100644 index 00000000..923fa8e5 --- /dev/null +++ b/tests/test_normalize_full_reference_bytes_literals.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_reference_bytes_literals as normalizer + + +def _helpers() -> dict[str, object]: + namespace: dict[str, object] = { + "_native_byte_data": [0], + "_native_kind_key": lambda runtime, name: f"g:{runtime}:{name}", + "_native_builder_key": lambda runtime, handle: f"h:{runtime}:{handle}", + } + exec(normalizer._HELPERS, namespace) + return namespace + + +def _payload(namespace: dict[str, object], key: str) -> list[int]: + arena = namespace["_native_byte_data"] + starts = namespace["_native_literal_start"] + sizes = namespace["_native_literal_size"] + start = starts[key] + size = sizes[key] + return arena[start : start + size] + + +def test_parses_bytes_literal_escapes_exactly() -> None: + namespace = _helpers() + store = namespace["_native_store_bytes_literal"] + + assert store("hex", 'b"\\x00\\xffA"') is True + assert _payload(namespace, "hex") == [0, 255, 65] + + assert store("mixed", "b'\\101\\n\\t\\\\\\\''") is True + assert _payload(namespace, "mixed") == [65, 10, 9, 92, 39] + + assert store("raw", 'rb"\\x41"') is True + assert _payload(namespace, "raw") == [92, 120, 52, 49] + + +def test_rejects_non_bytes_and_invalid_hex_literals() -> None: + namespace = _helpers() + store = namespace["_native_store_bytes_literal"] + + assert store("text", '"abc"') is False + assert store("bad", 'b"\\xz0"') is False + assert "text" not in namespace["_native_literal_start"] + assert "bad" not in namespace["_native_literal_start"] + + +def test_attaches_global_and_expression_payloads_to_handles() -> None: + namespace = _helpers() + record = namespace["_native_record_global_bytes"] + attach_global = namespace["_native_attach_global_bytes"] + attach_expression = namespace["_native_attach_expression_bytes"] + + record(4, "payload", 'b"\\x00\\xffA"') + attach_global(4, "payload", 9) + assert _payload(namespace, "h:4:9") == [0, 255, 65] + + attach_expression(4, 'b"eval"', 10) + assert _payload(namespace, "h:4:10") == [101, 118, 97, 108] + + +def test_installs_bytes_literal_ledger_into_native_abi( + tmp_path: Path, monkeypatch, +) -> None: + output = tmp_path / "native_full_reference_entry.py" + output.write_text( + '''def _native_record_statement_kind(runtime, name, text, equals, kind): + _native_set_global_kind(runtime, name, kind) + + +def _portapy_get_global_span_impl(runtime, name_text, value, kind): + if value: + _native_set_handle_kind(runtime, value, kind) + + +def _portapy_eval_span_impl(runtime, source_text, value, kind): + if value: + _native_set_handle_kind(runtime, value, kind) + + +def _native_data_size(runtime, handle, kind, value): + return 0 + + +def _native_data_byte(runtime, handle, kind, value, index): + return 0 +''', + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", output) + + assert normalizer.main() == 0 + + module = ast.parse(output.read_text(encoding="utf-8")) + text = ast.unparse(module) + assert "_native_record_global_bytes(runtime, name, text[equals + 1:])" in text + assert "_native_attach_global_bytes(runtime, name_text, value)" in text + assert "_native_attach_expression_bytes(runtime, source_text, value)" in text + assert "literal_start = _native_literal_start.get(literal_key, -1)" in text + assert "_native_byte_data[literal_start + index]" in text diff --git a/tests/test_normalize_full_reference_data_access.py b/tests/test_normalize_full_reference_data_access.py new file mode 100644 index 00000000..f54e5ff8 --- /dev/null +++ b/tests/test_normalize_full_reference_data_access.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import materialize_full_reference_entry as materializer +from tools import normalize_full_reference_abi_helpers as abi_normalizer +from tools import normalize_full_reference_data_access as access_normalizer +from tools import normalize_full_reference_data_builders as builder_normalizer +from tools import normalize_full_reference_errors as error_normalizer +from tools import normalize_full_reference_float_bits as float_normalizer + + +class _DataBuilder: + pass + + +class _Kinds: + STRING = object() + BYTES = object() + + +def _helpers() -> dict[str, object]: + namespace: dict[str, object] = { + "_DataBuilder": _DataBuilder, + "ValueKind": _Kinds, + "_native_byte_data": [0], + } + exec(access_normalizer._HELPERS, namespace) + return namespace + + +def test_calculates_utf8_size_and_bytes_without_introspection() -> None: + namespace = _helpers() + size = namespace["_native_data_size"] + byte = namespace["_native_data_byte"] + value = "Aπ😀" + expected = value.encode("utf-8") + assert size(1, 2, _Kinds.STRING, value) == len(expected) + assert bytes( + byte(1, 2, _Kinds.STRING, value, index) + for index in range(len(expected)) + ) == expected + raw = b"\x00\xffA" + assert size(1, 3, _Kinds.BYTES, raw) == 3 + assert [ + byte(1, 3, _Kinds.BYTES, raw, index) + for index in range(3) + ] == [0, 255, 65] + + +def test_builder_handle_uses_arena_without_payload_type_checks() -> None: + namespace = _helpers() + size = namespace["_native_data_size"] + byte = namespace["_native_data_byte"] + key = namespace["_native_builder_key"] + markers = namespace["_native_builder_handles"] + arena = namespace["_native_byte_data"] + builder = _DataBuilder() + builder.start = len(arena) + builder.size = 2 + builder.written = 2 + arena.extend([10, 255]) + markers[key(4, 7)] = True + assert size(4, 7, _Kinds.STRING, builder) == 2 + assert byte(4, 7, _Kinds.STRING, builder, 0) == 10 + assert byte(4, 7, _Kinds.STRING, builder, 1) == 255 + + +def test_installs_direct_data_access_into_native_abi( + tmp_path: Path, monkeypatch, +) -> None: + output = tmp_path / "native_full_reference_entry.py" + monkeypatch.setattr(materializer, "OUTPUT", output) + monkeypatch.setattr(abi_normalizer, "PATH", output) + monkeypatch.setattr(float_normalizer, "PATH", output) + monkeypatch.setattr(error_normalizer, "PATH", output) + monkeypatch.setattr(builder_normalizer, "PATH", output) + monkeypatch.setattr(access_normalizer, "PATH", output) + + assert materializer.main() == 0 + assert abi_normalizer.main() == 0 + assert float_normalizer.main() == 0 + assert error_normalizer.main() == 0 + assert builder_normalizer.main() == 0 + assert access_normalizer.main() == 0 + + module = ast.parse(output.read_text(encoding="utf-8")) + text = ast.unparse(module) + assert "_native_builder_handles[_native_builder_key(runtime, result)] = True" in text + assert "_native_string_byte(value, index)" in text + assert "size = _native_data_size(runtime, value, kind, raw)" in text + assert "result = _native_data_byte(runtime, value, kind, raw, index)" in text + functions = "\n".join( + ast.unparse(node) + for node in module.body + if isinstance(node, ast.FunctionDef) + and node.name in { + "_portapy_value_get_size_impl", + "_portapy_value_get_byte_impl", + } + ) + assert "type(raw)" not in functions + assert "isinstance(raw" not in functions diff --git a/tests/test_normalize_full_reference_data_builders.py b/tests/test_normalize_full_reference_data_builders.py new file mode 100644 index 00000000..24426d8d --- /dev/null +++ b/tests/test_normalize_full_reference_data_builders.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import materialize_full_reference_entry as materializer +from tools import normalize_full_reference_abi_helpers as abi_normalizer +from tools import normalize_full_reference_data_builders as builder_normalizer +from tools import normalize_full_reference_errors as error_normalizer +from tools import normalize_full_reference_float_bits as float_normalizer + + +def _function(module: ast.Module, name: str) -> str: + node = next( + item + for item in module.body + if isinstance(item, ast.FunctionDef) and item.name == name + ) + return ast.unparse(node) + + +def test_installs_sequential_native_data_builders( + tmp_path: Path, monkeypatch, +) -> None: + output = tmp_path / "native_full_reference_entry.py" + monkeypatch.setattr(materializer, "OUTPUT", output) + monkeypatch.setattr(abi_normalizer, "PATH", output) + monkeypatch.setattr(float_normalizer, "PATH", output) + monkeypatch.setattr(error_normalizer, "PATH", output) + monkeypatch.setattr(builder_normalizer, "PATH", output) + + assert materializer.main() == 0 + assert abi_normalizer.main() == 0 + assert float_normalizer.main() == 0 + assert error_normalizer.main() == 0 + assert builder_normalizer.main() == 0 + + module = ast.parse(output.read_text(encoding="utf-8")) + text = ast.unparse(module) + assert "_native_byte_data: list[int] = [0]" in text + + builder = next( + node + for node in module.body + if isinstance(node, ast.ClassDef) and node.name == "_DataBuilder" + ) + builder_text = ast.unparse(builder) + assert "self.size = size" in builder_text + assert "self.start = len(_native_byte_data)" in builder_text + assert "self.written = 0" in builder_text + assert "self.data" not in builder_text + + materialize = _function(module, "_data_bytes") + assert "value.written != value.size" in materialize + assert "data: list[int] = [_native_byte_data[value.start]]" in materialize + assert "data.append(_native_byte_data[value.start + index])" in materialize + + begin = _function(module, "_portapy_value_from_data_begin_impl") + assert "instance._store(_DataBuilder(kind, size), _native_kind_member(kind))" in begin + + setter = _function(module, "_portapy_value_set_data_byte_impl") + assert "index != target.written" in setter + assert "_native_byte_data.append(byte)" in setter + assert "target.written += 1" in setter + assert "target.data" not in setter + + validator = _function(module, "_portapy_value_validate_utf8_impl") + assert "while index < raw.size" in validator + assert "invalid UTF-8 leading byte" in validator + assert "invalid UTF-8 continuation byte" in validator + assert "codepoint > 1114111" in validator + assert ".decode(" not in validator diff --git a/tests/test_normalize_full_reference_error_locations.py b/tests/test_normalize_full_reference_error_locations.py new file mode 100644 index 00000000..d6a81dbf --- /dev/null +++ b/tests/test_normalize_full_reference_error_locations.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import materialize_full_reference_entry as materializer +from tools import normalize_full_reference_abi_helpers as abi_normalizer +from tools import normalize_full_reference_error_locations as location_normalizer +from tools import normalize_full_reference_errors as error_normalizer +from tools import normalize_full_reference_float_bits as float_normalizer + + +def _locator(): + namespace: dict[str, object] = {} + source = "def locate(source: str):\n" + "\n".join( + " " + line for line in location_normalizer._LOCATION_BODY.splitlines() + ) + exec(source, namespace) + return namespace["locate"] + + +def _is_name_plus_one(node: ast.AST, name: str) -> bool: + return ( + isinstance(node, ast.BinOp) + and isinstance(node.op, ast.Add) + and isinstance(node.left, ast.Name) + and node.left.id == name + and isinstance(node.right, ast.Constant) + and node.right.value == 1 + ) + + +def _has_return_pair(function: ast.FunctionDef, second_name: str) -> bool: + return any( + isinstance(node, ast.Return) + and isinstance(node.value, ast.Tuple) + and len(node.value.elts) == 2 + and _is_name_plus_one(node.value.elts[0], "line_index") + and _is_name_plus_one(node.value.elts[1], second_name) + for node in ast.walk(function) + ) + + +def test_finds_invalid_indentation_and_zero_division() -> None: + locate = _locator() + assert locate("value = 1\n unexpected = 2\n") == (2, 3) + assert locate("if ready:\n value = 1\nresult = 2\n") == (1, 1) + assert locate("safe = 1\nbroken = 5 // 0") == (2, 12) + assert locate("value = 9 % 0") == (1, 11) + assert locate("text = '5 // 0'") == (1, 1) + + +def test_replaces_generated_error_locator( + tmp_path: Path, monkeypatch, +) -> None: + output = tmp_path / "native_full_reference_entry.py" + monkeypatch.setattr(materializer, "OUTPUT", output) + monkeypatch.setattr(abi_normalizer, "PATH", output) + monkeypatch.setattr(float_normalizer, "PATH", output) + monkeypatch.setattr(error_normalizer, "PATH", output) + monkeypatch.setattr(location_normalizer, "PATH", output) + + assert materializer.main() == 0 + assert abi_normalizer.main() == 0 + assert float_normalizer.main() == 0 + assert error_normalizer.main() == 0 + assert location_normalizer.main() == 0 + + module = ast.parse(output.read_text(encoding="utf-8")) + locator = next( + node + for node in module.body + if isinstance(node, ast.FunctionDef) + and node.name == "_native_error_location" + ) + names = { + node.id + for node in ast.walk(locator) + if isinstance(node, ast.Name) + } + assert "previous_opens_block" in names + assert _has_return_pair(locator, "indent") + assert _has_return_pair(locator, "column_index") diff --git a/tests/test_normalize_full_reference_error_text.py b/tests/test_normalize_full_reference_error_text.py new file mode 100644 index 00000000..4d9cde92 --- /dev/null +++ b/tests/test_normalize_full_reference_error_text.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import materialize_full_reference_entry as materializer +from tools import normalize_full_reference_abi_helpers as abi_normalizer +from tools import normalize_full_reference_error_text as text_normalizer + + +def _function(module: ast.Module, name: str) -> str: + return ast.unparse( + next( + node + for node in module.body + if isinstance(node, ast.FunctionDef) and node.name == name + ) + ) + + +def test_rewrites_error_text_helpers_without_encoding( + tmp_path: Path, monkeypatch, +) -> None: + output = tmp_path / "native_full_reference_entry.py" + monkeypatch.setattr(materializer, "OUTPUT", output) + monkeypatch.setattr(abi_normalizer, "PATH", output) + monkeypatch.setattr(text_normalizer, "PATH", output) + + assert materializer.main() == 0 + assert abi_normalizer.main() == 0 + assert text_normalizer.main() == 0 + + module = ast.parse(output.read_text(encoding="utf-8")) + type_size = _function(module, "_portapy_error_type_size_impl") + message_size = _function(module, "_portapy_error_message_size_impl") + type_byte = _function(module, "_portapy_error_type_byte_impl") + message_byte = _function(module, "_portapy_error_message_byte_impl") + + assert "len(error.type_name)" in type_size + assert "len(error.message)" in message_size + assert "ord(error.type_name[index])" in type_byte + assert "ord(error.message[index])" in message_byte + combined = type_size + message_size + type_byte + message_byte + assert ".encode(" not in combined + assert "_error_bytes(" not in combined diff --git a/tests/test_normalize_full_reference_errors.py b/tests/test_normalize_full_reference_errors.py new file mode 100644 index 00000000..98671261 --- /dev/null +++ b/tests/test_normalize_full_reference_errors.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import materialize_full_reference_entry as materializer +from tools import normalize_full_reference_abi_helpers as abi_normalizer +from tools import normalize_full_reference_errors as error_normalizer + + +def _function(module: ast.Module, name: str) -> str: + node = next( + item + for item in module.body + if isinstance(item, ast.FunctionDef) and item.name == name + ) + return ast.unparse(node) + + +def test_installs_native_structured_error_paths( + tmp_path: Path, monkeypatch, +) -> None: + output = tmp_path / "native_full_reference_entry.py" + monkeypatch.setattr(materializer, "OUTPUT", output) + monkeypatch.setattr(abi_normalizer, "PATH", output) + monkeypatch.setattr(error_normalizer, "PATH", output) + + assert materializer.main() == 0 + assert abi_normalizer.main() == 0 + assert error_normalizer.main() == 0 + + module = ast.parse(output.read_text(encoding="utf-8")) + validate = _function(module, "_portapy_value_validate_utf8_impl") + assert "UnicodeDecodeError" in validate + assert "instance._capture_native(" in validate + + execute = _function(module, "_portapy_exec_span_impl") + evaluate = _function(module, "_portapy_eval_span_impl") + assert "_native_error_location(source_text)" in execute + assert "_native_error_location(source_text)" in evaluate + assert "RuntimeError" in execute + assert "SyntaxError" in evaluate + + line = _function(module, "_portapy_error_line_impl") + column = _function(module, "_portapy_error_column_impl") + assert "return instance._error_line" in line + assert "return instance._error_column" in column + + +def test_native_error_location_finds_division_by_zero() -> None: + namespace: dict[str, object] = {} + exec(error_normalizer._LOCATION_HELPER, namespace) + locate = namespace["_native_error_location"] + + assert locate("safe = 1\nbroken = 5 // 0") == (2, 12) + assert locate("value = 9 % 0") == (1, 11) + assert locate("value = 4") == (1, 1) diff --git a/tests/test_normalize_full_reference_expression_kinds.py b/tests/test_normalize_full_reference_expression_kinds.py new file mode 100644 index 00000000..0ec0090c --- /dev/null +++ b/tests/test_normalize_full_reference_expression_kinds.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from pathlib import Path + +from tools import normalize_full_reference_expression_kinds as normalizer +from tools import normalize_full_reference_value_kinds as value_kinds + + +_PRELUDE = ''' +PORTAPY_VALUE_NONE = 0 +PORTAPY_VALUE_BOOL = 1 +PORTAPY_VALUE_INT = 2 +PORTAPY_VALUE_FLOAT = 3 +PORTAPY_VALUE_STRING = 4 +PORTAPY_VALUE_BYTES = 5 +PORTAPY_VALUE_CALLABLE = 6 +PORTAPY_VALUE_OBJECT = 7 +PORTAPY_VALUE_TUPLE = 8 +PORTAPY_VALUE_DICT = 9 +PORTAPY_VALUE_LIST = 10 + +class Runtime: + pass + +class ValueKind: + NONE = object() + BOOL = object() + INT = object() + FLOAT = object() + STRING = object() + BYTES = object() + CALLABLE = object() + OBJECT = object() + TUPLE = object() + DICT = object() + LIST = object() + +''' + + +def _normalized_helpers(tmp_path: Path, monkeypatch) -> dict[str, object]: + path = tmp_path / "native_full_reference_entry.py" + path.write_text(_PRELUDE + value_kinds._KIND_HELPERS, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + namespace: dict[str, object] = {} + exec(path.read_text(encoding="utf-8"), namespace) + return namespace + + +def test_literal_left_comparison_is_boolean(tmp_path: Path, monkeypatch) -> None: + namespace = _normalized_helpers(tmp_path, monkeypatch) + infer = namespace["_native_expression_kind"] + + assert infer(1, '"abc" < "abd"') == 1 + assert infer(1, '("abc" < "abd")') == 1 + assert infer(1, "'a < b'") == 4 + + +def test_boolops_keep_operand_kinds(tmp_path: Path, monkeypatch) -> None: + namespace = _normalized_helpers(tmp_path, monkeypatch) + infer = namespace["_native_expression_kind"] + set_kind = namespace["_native_set_global_kind"] + + set_kind(7, "empty", 4) + set_kind(7, "name", 4) + set_kind(7, "answer", 2) + + assert infer(7, "empty or name") == 4 + assert infer(7, "name and answer") == 2 + assert infer(7, 'answer > 40 and name == "Somnia"') == 1 + assert infer(7, "not empty") == 1 + + +def test_source_functions_propagate_return_kinds(tmp_path: Path, monkeypatch) -> None: + namespace = _normalized_helpers(tmp_path, monkeypatch) + infer = namespace["_native_expression_kind"] + record = namespace["_native_record_source_kinds"] + global_kind = namespace["_native_global_kind"] + + source = '''def seven(): + return 7 +def label(): + return "ready" +def make_items(): + return [1, 2] +class Box: + pass +answer = seven() +text = label() +items = make_items() +box = Box() +''' + record(9, source) + + assert infer(9, "seven()") == 2 + assert infer(9, "label()") == 4 + assert infer(9, "make_items()") == 10 + assert infer(9, "Box()") == 7 + assert global_kind(9, "answer") == 2 + assert global_kind(9, "text") == 4 + assert global_kind(9, "items") == 10 + assert global_kind(9, "box") == 7 + + +def test_nested_function_return_is_callable(tmp_path: Path, monkeypatch) -> None: + namespace = _normalized_helpers(tmp_path, monkeypatch) + infer = namespace["_native_expression_kind"] + record = namespace["_native_record_source_kinds"] + + source = '''def outer(base): + def inner(value): + return base + value + return inner +fn = outer(19) +''' + record(11, source) + + assert infer(11, "outer(19)") == 6 + assert infer(11, "fn") == 6 diff --git a/tests/test_normalize_full_reference_float_bits.py b/tests/test_normalize_full_reference_float_bits.py new file mode 100644 index 00000000..58def803 --- /dev/null +++ b/tests/test_normalize_full_reference_float_bits.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import materialize_full_reference_entry as materializer +from tools import normalize_full_reference_abi_helpers as abi_normalizer +from tools import normalize_full_reference_float_bits as float_normalizer + + +def _function(module: ast.Module, name: str) -> ast.FunctionDef: + return next( + node + for node in module.body + if isinstance(node, ast.FunctionDef) and node.name == name + ) + + +def test_rewrites_native_float_functions_to_integer_bits( + tmp_path: Path, monkeypatch, +) -> None: + output = tmp_path / "native_full_reference_entry.py" + monkeypatch.setattr(materializer, "OUTPUT", output) + monkeypatch.setattr(abi_normalizer, "PATH", output) + monkeypatch.setattr(float_normalizer, "PATH", output) + + assert materializer.main() == 0 + assert abi_normalizer.main() == 0 + assert float_normalizer.main() == 0 + + module = ast.parse(output.read_text(encoding="utf-8")) + names = { + node.name + for node in module.body + if isinstance(node, ast.FunctionDef) + } + assert "_portapy_value_from_f64_impl" not in names + assert "_portapy_value_as_f64_impl" not in names + + constructor = _function(module, "_portapy_value_from_f64_bits_impl") + conversion = _function(module, "_portapy_value_as_f64_bits_impl") + assert constructor.args.args[1].arg == "bits" + assert ast.unparse(constructor.args.args[1].annotation) == "int" + assert "instance._store(bits, ValueKind.FLOAT)" in ast.unparse(constructor) + assert "kind is not ValueKind.FLOAT" in ast.unparse(conversion) + assert "instance.unbox(value)" in ast.unparse(conversion) diff --git a/tests/test_normalize_full_reference_function_return_kinds.py b/tests/test_normalize_full_reference_function_return_kinds.py new file mode 100644 index 00000000..d6bdb197 --- /dev/null +++ b/tests/test_normalize_full_reference_function_return_kinds.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from pathlib import Path + +from tools import normalize_full_reference_expression_kinds as expression_kinds +from tools import normalize_full_reference_function_return_kinds as return_kinds +from tools import normalize_full_reference_value_kinds as value_kinds + + +_PRELUDE = ''' +PORTAPY_VALUE_NONE = 0 +PORTAPY_VALUE_BOOL = 1 +PORTAPY_VALUE_INT = 2 +PORTAPY_VALUE_FLOAT = 3 +PORTAPY_VALUE_STRING = 4 +PORTAPY_VALUE_BYTES = 5 +PORTAPY_VALUE_CALLABLE = 6 +PORTAPY_VALUE_OBJECT = 7 +PORTAPY_VALUE_TUPLE = 8 +PORTAPY_VALUE_DICT = 9 +PORTAPY_VALUE_LIST = 10 + +class Runtime: + pass + +class ValueKind: + NONE = object() + BOOL = object() + INT = object() + FLOAT = object() + STRING = object() + BYTES = object() + CALLABLE = object() + OBJECT = object() + TUPLE = object() + DICT = object() + LIST = object() + +''' + + +def _normalized_namespace(tmp_path: Path, monkeypatch) -> dict[str, object]: + path = tmp_path / "native_full_reference_entry.py" + path.write_text(_PRELUDE + value_kinds._KIND_HELPERS, encoding="utf-8") + monkeypatch.setattr(expression_kinds, "PATH", path) + monkeypatch.setattr(return_kinds, "PATH", path) + + assert expression_kinds.main() == 0 + assert return_kinds.main() == 0 + + namespace: dict[str, object] = {} + exec(path.read_text(encoding="utf-8"), namespace) + return namespace + + +def test_mixed_return_kinds_stay_object(tmp_path: Path, monkeypatch) -> None: + namespace = _normalized_namespace(tmp_path, monkeypatch) + record = namespace["_native_record_source_kinds"] + infer = namespace["_native_expression_kind"] + + record( + 1, + '''def mixed(flag): + if flag: + return 7 + return + +def never_unmix(flag): + if flag == 1: + return 7 + if flag == 2: + return "seven" + return 8 +''', + ) + + assert infer(1, "mixed(True)") == 7 + assert infer(1, "never_unmix(1)") == 7 + + +def test_consistent_and_none_returns_remain_precise( + tmp_path: Path, + monkeypatch, +) -> None: + namespace = _normalized_namespace(tmp_path, monkeypatch) + record = namespace["_native_record_source_kinds"] + infer = namespace["_native_expression_kind"] + + record( + 2, + '''def consistent(flag): + if flag: + return 7 + return 8 + +def explicit_none(): + return + +def no_return(): + pass +''', + ) + + assert infer(2, "consistent(True)") == 2 + assert infer(2, "explicit_none()") == 0 + assert infer(2, "no_return()") == 7 + + +def test_nested_callable_return_uses_single_ledger( + tmp_path: Path, + monkeypatch, +) -> None: + namespace = _normalized_namespace(tmp_path, monkeypatch) + record = namespace["_native_record_source_kinds"] + infer = namespace["_native_expression_kind"] + + record( + 3, + '''def outer(): + def inner(): + return 42 + return inner +''', + ) + + assert infer(3, "outer()") == 6 + assert "_native_function_return_kinds" not in namespace + assert "_native_record_function_return_kinds" not in namespace diff --git a/tests/test_normalize_full_reference_handle_kind_access.py b/tests/test_normalize_full_reference_handle_kind_access.py new file mode 100644 index 00000000..4915ab56 --- /dev/null +++ b/tests/test_normalize_full_reference_handle_kind_access.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import normalize_full_reference_handle_kind_access as normalizer + + +SOURCE = '''def _native_set_handle_kind(instance: Runtime, handle: int, kind: int) -> bool: + slot = instance._values.get(str(handle)) + if slot is None: + return False + slot.kind = _native_kind_member(kind) + return True +''' + + +def test_uses_normalized_value_slot_accessor( + tmp_path: Path, + monkeypatch, +) -> None: + path = tmp_path / "native_full_reference_entry.py" + path.write_text(SOURCE, encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + source = path.read_text(encoding="utf-8") + assert "slot = instance._value_slot(handle)" in source + assert "slot.kind = _native_kind_member(kind)" in source + assert "instance._values" not in source + assert ".get(str(handle))" not in source + ast.parse(source) + + +def test_fails_closed_when_stale_shape_changes( + tmp_path: Path, + monkeypatch, +) -> None: + path = tmp_path / "native_full_reference_entry.py" + path.write_text( + SOURCE.replace( + "instance._values.get(str(handle))", + "instance._value_slot(handle)", + ), + encoding="utf-8", + ) + monkeypatch.setattr(normalizer, "PATH", path) + + try: + normalizer.main() + except RuntimeError as error: + assert "stale dict lookup" in str(error) + else: + raise AssertionError("normalizer accepted an unexpected helper shape") diff --git a/tests/test_normalize_full_reference_native_call_alignment.py b/tests/test_normalize_full_reference_native_call_alignment.py new file mode 100644 index 00000000..37125923 --- /dev/null +++ b/tests/test_normalize_full_reference_native_call_alignment.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import ast + +from tools import normalize_full_reference_safe_host_ids as normalizer + + +def test_legacy_value_lookup_uses_native_slot_accessor() -> None: + module = ast.parse( + ''' +def replace_value(instance, handle, value): + slot = instance._values.get(handle) + slot.value = value +''' + ) + rewrite = normalizer._LegacyValueLookupRewriter() + module = rewrite.visit(module) + ast.fix_missing_locations(module) + source = ast.unparse(module) + + assert rewrite.replaced == 1 + assert "slot = instance._value_slot(handle)" in source + assert "instance._values.get(handle)" not in source + + +def test_nested_slot_value_calls_are_hoisted_before_container_writes() -> None: + module = ast.parse( + ''' +def assign_item(instance, item, values, index): + values[index] = _slot_value(instance, item) + + +def append_item(instance, item, values): + values.append(_slot_value(instance, item)) +''' + ) + hoister = normalizer._NestedSlotValueHoister() + module = hoister.visit(module) + ast.fix_missing_locations(module) + source = ast.unparse(module) + + assert hoister.hoisted == 2 + assert "values[index] = _slot_value(" not in source + assert "values.append(_slot_value(" not in source + assert "_native_slot_value_0 = _slot_value(instance, item)" in source + assert "values[index] = _native_slot_value_0" in source + assert "_native_slot_value_1 = _slot_value(instance, item)" in source + assert "values.append(_native_slot_value_1)" in source diff --git a/tests/test_normalize_full_reference_nested_kinds.py b/tests/test_normalize_full_reference_nested_kinds.py new file mode 100644 index 00000000..eb14d23b --- /dev/null +++ b/tests/test_normalize_full_reference_nested_kinds.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import materialize_full_reference_entry as materializer +from tools import normalize_full_reference_abi_helpers as abi_normalizer +from tools import normalize_full_reference_errors as error_normalizer +from tools import normalize_full_reference_float_bits as float_normalizer +from tools import normalize_full_reference_nested_kinds as nested_normalizer +from tools import normalize_full_reference_value_kinds as kind_normalizer + + +def test_removes_top_level_only_kind_guard( + tmp_path: Path, monkeypatch, +) -> None: + output = tmp_path / "native_full_reference_entry.py" + monkeypatch.setattr(materializer, "OUTPUT", output) + monkeypatch.setattr(abi_normalizer, "PATH", output) + monkeypatch.setattr(float_normalizer, "PATH", output) + monkeypatch.setattr(error_normalizer, "PATH", output) + monkeypatch.setattr(kind_normalizer, "PATH", output) + monkeypatch.setattr(nested_normalizer, "PATH", output) + + assert materializer.main() == 0 + assert abi_normalizer.main() == 0 + assert float_normalizer.main() == 0 + assert error_normalizer.main() == 0 + assert kind_normalizer.main() == 0 + assert nested_normalizer.main() == 0 + + module = ast.parse(output.read_text(encoding="utf-8")) + scanner = next( + node + for node in module.body + if isinstance(node, ast.FunctionDef) + and node.name == "_native_record_source_kinds" + ) + text = ast.unparse(scanner) + assert "if indentation == 0" not in text + assert text.count("_native_record_statement_kind(runtime, statement)") == 1 diff --git a/tests/test_normalize_full_reference_runtime.py b/tests/test_normalize_full_reference_runtime.py new file mode 100644 index 00000000..f55c01a0 --- /dev/null +++ b/tests/test_normalize_full_reference_runtime.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tools import normalize_full_reference_runtime as normalizer + + +REFERENCE_SOURCE = Path("src/portapy/reference_api.py") + + +def test_normalizes_real_reference_runtime( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "reference_api.py" + path.write_text(REFERENCE_SOURCE.read_text(encoding="utf-8"), encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + + result = path.read_text(encoding="utf-8") + assert "import traceback" not in result + assert "format_exception" not in result + assert "type(error).__name__" not in result + assert "str(error)" not in result + assert '"PortaPyError"' in result + assert result.count('"PortaPy operation failed"') == 2 + + assert "def _capture_native(" in result + assert "ErrorInfo(status, type_name, message, message)" in result + assert "self._error_line = line" in result + assert "self._error_column = column" in result + assert result.count("self._error_line = 0") >= 4 + assert result.count("self._error_column = 0") >= 4 + + assert "self._values: list[_Slot | None] = [None]" in result + assert "kind: ValueKind = ValueKind.INT" in result + assert "self._values.append(_Slot(value, kind))" in result + assert "def _value_slot(self, handle: int) -> _Slot | None:" in result + assert "if handle <= 0 or handle >= len(self._values):" in result + assert result.count("self._value_slot(handle)") == 8 + assert result.count("self._value_slot(callable_handle)") == 1 + assert "self._values[handle] = None" in result + assert "str(handle)" not in result + assert "dict[str, _Slot]" not in result + + assert "self._store(None, ValueKind.NONE)" in result + assert "self._store(value, ValueKind.BOOL)" in result + assert "self._store(value, ValueKind.INT)" in result + assert "self._store(value, ValueKind.FLOAT)" in result + assert "self._store(value, ValueKind.STRING)" in result + assert "self._store(value, ValueKind.BYTES)" in result + + assert "return Status.OK, slot.kind" in result + assert "slot.kind is not ValueKind.INT" in result + assert "slot.kind is not ValueKind.FLOAT" in result + assert "slot.kind is not ValueKind.STRING" in result + assert "type(slot.value)" not in result + + +def test_native_value_slots_use_stable_integer_indices( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "reference_api.py" + path.write_text(REFERENCE_SOURCE.read_text(encoding="utf-8"), encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + + assert normalizer.main() == 0 + namespace: dict[str, object] = {} + # The complete module has package-relative imports, so execute only the + # normalized Runtime storage methods in a tiny compatible shell. + source = path.read_text(encoding="utf-8") + runtime_start = source.index("class Runtime:") + runtime_source = source[runtime_start:] + prefix = ''' +from dataclasses import dataclass +from enum import IntEnum +class Status(IntEnum): + OK = 0 + INVALID_HANDLE = 7 +class ValueKind(IntEnum): + INT = 2 +@dataclass +class _Slot: + value: object + kind: ValueKind = ValueKind.INT + refs: int = 1 +''' + # Keep just the methods needed for storage semantics. + store_start = runtime_source.index(" def _store(") + close_start = runtime_source.index(" def close(") + methods = runtime_source[store_start:close_start] + exec(prefix + "\nclass Probe:\n" + methods, namespace) + probe = namespace["Probe"]() + probe._values = [None] + probe._next = 1 + first = probe._store("first") + second = probe._store("second") + assert (first, second) == (1, 2) + assert probe._value_slot(first).value == "first" + assert probe._value_slot(second).value == "second" + assert probe._value_slot(0) is None + assert probe._value_slot(99) is None + + +def test_rejects_unexpected_reference_runtime_shape( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "reference_api.py" + path.write_text("import traceback\n", encoding="utf-8") + monkeypatch.setattr(normalizer, "PATH", path) + with pytest.raises(RuntimeError, match="error capture normalization"): + normalizer.main() diff --git a/tests/test_normalize_full_reference_source_preprocess.py b/tests/test_normalize_full_reference_source_preprocess.py new file mode 100644 index 00000000..478e95f2 --- /dev/null +++ b/tests/test_normalize_full_reference_source_preprocess.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import materialize_full_reference_entry as materializer +from tools import normalize_full_reference_abi_helpers as abi_normalizer +from tools import normalize_full_reference_errors as error_normalizer +from tools import normalize_full_reference_float_bits as float_normalizer +from tools import normalize_full_reference_source_preprocess as source_normalizer +from tools import normalize_full_reference_value_kinds as kind_normalizer + + +def _expand(source: str) -> str: + namespace: dict[str, object] = {} + exec(source_normalizer._HELPER_SOURCE, namespace) + return namespace["_native_expand_runtime_source"](source) + + +def test_expands_top_level_semicolons_without_touching_literals_or_comments() -> None: + assert _expand("first = 1; second = 2") == "first = 1\nsecond = 2" + assert _expand("text = 'a;b#c'; answer = 42 # ; ignored") == ( + "text = 'a;b#c'\nanswer = 42 # ; ignored" + ) + assert _expand("items = [1; 2]") == "items = [1; 2]" + + +def test_indents_compact_compound_suites() -> None: + assert _expand("if flag: first = 1; second = 2") == ( + "if flag: first = 1\n second = 2" + ) + assert _expand("while ready: tick(); stop()") == ( + "while ready: tick()\n stop()" + ) + + +def test_consumes_tabs_and_spaces_after_separator() -> None: + assert _expand("first = 1;\t second = 2") == "first = 1\nsecond = 2" + assert _expand("if flag: first = 1;\t second = 2") == ( + "if flag: first = 1\n second = 2" + ) + + +def test_installs_preprocessing_into_exec_and_eval( + tmp_path: Path, monkeypatch, +) -> None: + output = tmp_path / "native_full_reference_entry.py" + monkeypatch.setattr(materializer, "OUTPUT", output) + monkeypatch.setattr(abi_normalizer, "PATH", output) + monkeypatch.setattr(float_normalizer, "PATH", output) + monkeypatch.setattr(error_normalizer, "PATH", output) + monkeypatch.setattr(kind_normalizer, "PATH", output) + monkeypatch.setattr(source_normalizer, "PATH", output) + + assert materializer.main() == 0 + assert abi_normalizer.main() == 0 + assert float_normalizer.main() == 0 + assert error_normalizer.main() == 0 + assert kind_normalizer.main() == 0 + assert source_normalizer.main() == 0 + + module = ast.parse(output.read_text(encoding="utf-8")) + text = ast.unparse(module) + assert text.count("_native_expand_runtime_source(source[0:source_size])") == 2 diff --git a/tests/test_normalize_full_reference_type_errors.py b/tests/test_normalize_full_reference_type_errors.py new file mode 100644 index 00000000..39e0cf63 --- /dev/null +++ b/tests/test_normalize_full_reference_type_errors.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +from tools import normalize_full_reference_runtime as runtime_normalizer +from tools import normalize_full_reference_type_errors as type_error_normalizer + + +REFERENCE_SOURCE = Path("src/portapy/reference_api.py") + + +def _is_vm_run_try(statement: ast.stmt) -> bool: + return isinstance(statement, ast.Try) and any( + isinstance(item, ast.Expr) + and isinstance(item.value, ast.Call) + and isinstance(item.value.func, ast.Attribute) + and item.value.func.attr == "run" + for item in statement.body + ) + + +def test_type_error_handler_precedes_base_exception( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "reference_api.py" + path.write_text(REFERENCE_SOURCE.read_text(encoding="utf-8"), encoding="utf-8") + monkeypatch.setattr(runtime_normalizer, "PATH", path) + monkeypatch.setattr(type_error_normalizer, "PATH", path) + + assert runtime_normalizer.main() == 0 + assert type_error_normalizer.main() == 0 + + module = ast.parse(path.read_text(encoding="utf-8")) + runtime = next( + node + for node in module.body + if isinstance(node, ast.ClassDef) and node.name == "Runtime" + ) + method = next( + node + for node in runtime.body + if isinstance(node, ast.FunctionDef) and node.name == "exec_utf8" + ) + run_try = next(statement for statement in method.body if _is_vm_run_try(statement)) + handler_names = [ + handler.type.id if isinstance(handler.type, ast.Name) else "" + for handler in run_try.handlers + ] + + assert handler_names.index("TypeError") < handler_names.index("BaseException") + type_handler = run_try.handlers[handler_names.index("TypeError")] + handler_source = ast.unparse(type_handler) + assert "Status.TYPE_ERROR" in handler_source + assert "self._capture_native(" in handler_source diff --git a/tests/test_normalize_full_reference_value_kinds.py b/tests/test_normalize_full_reference_value_kinds.py new file mode 100644 index 00000000..fcd956f4 --- /dev/null +++ b/tests/test_normalize_full_reference_value_kinds.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from tools import materialize_full_reference_entry as materializer +from tools import normalize_full_reference_abi_helpers as abi_normalizer +from tools import normalize_full_reference_errors as error_normalizer +from tools import normalize_full_reference_float_bits as float_normalizer +from tools import normalize_full_reference_value_kinds as kind_normalizer + + +_CONSTANTS = { + "PORTAPY_VALUE_NONE": 0, + "PORTAPY_VALUE_BOOL": 1, + "PORTAPY_VALUE_INT": 2, + "PORTAPY_VALUE_FLOAT": 3, + "PORTAPY_VALUE_STRING": 4, + "PORTAPY_VALUE_BYTES": 5, + "PORTAPY_VALUE_CALLABLE": 6, + "PORTAPY_VALUE_OBJECT": 7, + "PORTAPY_VALUE_TUPLE": 8, + "PORTAPY_VALUE_DICT": 9, + "PORTAPY_VALUE_LIST": 10, +} + + +class _Kinds: + NONE = object() + BOOL = object() + INT = object() + FLOAT = object() + STRING = object() + BYTES = object() + CALLABLE = object() + OBJECT = object() + TUPLE = object() + DICT = object() + LIST = object() + + +def _helpers() -> dict[str, object]: + namespace: dict[str, object] = dict(_CONSTANTS) + namespace["Runtime"] = object + namespace["ValueKind"] = _Kinds + exec(kind_normalizer._KIND_HELPERS, namespace) + return namespace + + +def test_records_top_level_literal_alias_and_callable_kinds() -> None: + namespace = _helpers() + record = namespace["_native_record_source_kinds"] + get_kind = namespace["_native_global_kind"] + record( + 3, + "nothing = None\n" + "flag = True\n" + "name = 'Somnia'\n" + "payload = b'\\x00A'\n" + "alias = name\n" + "punctuation = 'a;b#c'; answer = 40 + 2\n" + "items = [1, 2]\n" + "pair = (1, 2)\n" + "mapping = {'x': 1}\n" + "def work(value):\n local = value\n return local\n" + "class Widget:\n pass\n", + ) + assert get_kind(3, "nothing") == 0 + assert get_kind(3, "flag") == 1 + assert get_kind(3, "name") == 4 + assert get_kind(3, "payload") == 5 + assert get_kind(3, "alias") == 4 + assert get_kind(3, "punctuation") == 4 + assert get_kind(3, "answer") == 2 + assert get_kind(3, "items") == 10 + assert get_kind(3, "pair") == 8 + assert get_kind(3, "mapping") == 9 + assert get_kind(3, "work") == 6 + assert get_kind(3, "Widget") == 6 + assert get_kind(3, "local") == 2 + + +def test_infers_eval_result_kinds() -> None: + namespace = _helpers() + infer = namespace["_native_expression_kind"] + set_kind = namespace["_native_set_global_kind"] + set_kind(1, "text", 4) + set_kind(1, "number", 2) + assert infer(1, "None") == 0 + assert infer(1, "False") == 1 + assert infer(1, "3.5") == 3 + assert infer(1, "text") == 4 + assert infer(1, "text + '!' ") == 4 + assert infer(1, "number + 8") == 2 + assert infer(1, "number > 1") == 1 + + +def test_installs_kind_ledger_into_native_abi( + tmp_path: Path, monkeypatch, +) -> None: + output = tmp_path / "native_full_reference_entry.py" + monkeypatch.setattr(materializer, "OUTPUT", output) + monkeypatch.setattr(abi_normalizer, "PATH", output) + monkeypatch.setattr(float_normalizer, "PATH", output) + monkeypatch.setattr(error_normalizer, "PATH", output) + monkeypatch.setattr(kind_normalizer, "PATH", output) + + assert materializer.main() == 0 + assert abi_normalizer.main() == 0 + assert float_normalizer.main() == 0 + assert error_normalizer.main() == 0 + assert kind_normalizer.main() == 0 + + module = ast.parse(output.read_text(encoding="utf-8")) + text = ast.unparse(module) + assert "_native_record_source_kinds(runtime, source_text)" in text + assert "_native_expression_kind(runtime, source_text)" in text + assert "_native_global_kind(runtime, name_text)" in text + assert "slot.kind = _native_kind_member(kind)" in text diff --git a/tests/test_release_gate.py b/tests/test_release_gate.py index 04664cc1..7c5adcbb 100644 --- a/tests/test_release_gate.py +++ b/tests/test_release_gate.py @@ -6,29 +6,29 @@ from tools.native_surface import public_exports from tools.python_surface import PYTHON_MODULE_EXPORTS -from tools.release_gate import main +from tools.release_gate import FULL_RUNTIME_FLAGS, main def _sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def test_release_gate_validates_both_native_artifacts(tmp_path: Path) -> None: - dist = tmp_path / "dist" - dist.mkdir() +def _write_artifacts(dist: Path, *, full_runtime: bool) -> None: expected_exports = list(public_exports(host_bridge=True, host_calls=True)) for target, name in (("linux", "libportapy.so"), ("windows", "portapy.dll")): artifact = dist / name - artifact.write_bytes((target.encode("ascii") + b"\0") * 1024) - if artifact.stat().st_size < 4096: - artifact.write_bytes(artifact.read_bytes() + b"x" * 4096) - metadata = { + artifact.write_bytes((target.encode("ascii") + b"\0") * 1024 + b"x" * 4096) + metadata: dict[str, object] = { "schema": 1, "target": target, "artifact": name, "size": artifact.stat().st_size, "sha256": _sha256(artifact), - "source": "src/portapy/native_api_host_calls.py", + "source": ( + "src/portapy/native_full_reference_entry.py" + if full_runtime + else "src/portapy/native_api_host_calls.py" + ), "source_sha256": "0" * 64, "public_exports": expected_exports, "python_module_exports": list(PYTHON_MODULE_EXPORTS), @@ -38,38 +38,57 @@ def test_release_gate_validates_both_native_artifacts(tmp_path: Path) -> None: "host_calls": True, "native_environment_adapter": True, "public_environment_api": True, - "generated_host_call_entry": True, + "generated_host_call_entry": not full_runtime, } + if full_runtime: + for flag in FULL_RUNTIME_FLAGS: + metadata[flag] = True artifact.with_suffix(artifact.suffix + ".json").write_text( - json.dumps(metadata), - encoding="utf-8", + json.dumps(metadata), encoding="utf-8" ) - status = tmp_path / "status.json" - status.write_text( + +def _write_status(path: Path, *, source_ready: bool) -> str: + tag = "3.14.0" if source_ready else "3.14-dev.1" + path.write_text( json.dumps( { "version_line": "3.14", - "release_tag": "3.14-dev.1", - "stage": "developer-preview", - "prerelease": True, + "release_tag": tag, + "stage": "stable" if source_ready else "developer-preview", + "prerelease": not source_ready, "python_built_runtime": True, - "source_execution_ready": False, + "source_execution_ready": source_ready, "completed_surface": ["runtime handles"], - "release_blockers": ["native parser"], + "release_blockers": [] if source_ready else ["native parser"], } ), encoding="utf-8", ) + return tag + + +def _run_gate(tmp_path: Path, *, source_ready: bool) -> Path: + dist = tmp_path / "dist" + dist.mkdir() + _write_artifacts(dist, full_runtime=source_ready) + status = tmp_path / "status.json" + tag = _write_status(status, source_ready=source_ready) + assert main([str(dist), "--status", str(status), "--expected-tag", tag]) == 0 + return dist + + +def test_release_gate_validates_preview_artifacts(tmp_path: Path) -> None: + dist = _run_gate(tmp_path, source_ready=False) + manifest = json.loads((dist / "release-manifest.json").read_text(encoding="utf-8")) + assert manifest["release"]["source_execution_ready"] is False + assert "Not yet included" in (dist / "RELEASE_NOTES.md").read_text(encoding="utf-8") + - assert main([str(dist), "--status", str(status), "--expected-tag", "3.14-dev.1"]) == 0 - assert (dist / "checksums.json").is_file() +def test_release_gate_validates_full_runtime_artifacts(tmp_path: Path) -> None: + dist = _run_gate(tmp_path, source_ready=True) manifest = json.loads((dist / "release-manifest.json").read_text(encoding="utf-8")) - assert manifest["release"]["stage"] == "developer-preview" - assert manifest["public_exports"] == expected_exports + assert manifest["release"]["source_execution_ready"] is True assert manifest["python_module_exports"] == list(PYTHON_MODULE_EXPORTS) - assert manifest["python_module_entry"] == "portapy" notes = (dist / "RELEASE_NOTES.md").read_text(encoding="utf-8") - assert "not the final Python 3.14 interpreter release" in notes - assert "`add_all`" in notes - assert "does not expose `import_module`" in notes + assert "Standalone source execution" in notes diff --git a/tools/build_native.py b/tools/build_native.py index cb248333..ad0afc8b 100644 --- a/tools/build_native.py +++ b/tools/build_native.py @@ -263,6 +263,7 @@ def build_native( "-shared", *link_objects, f"-Wl,--version-script={version_script}", + "-lm", "-o", str(output), ] diff --git a/tools/build_native_full_runtime.py b/tools/build_native_full_runtime.py index 193e0013..b3f3c586 100644 --- a/tools/build_native_full_runtime.py +++ b/tools/build_native_full_runtime.py @@ -4,6 +4,8 @@ import argparse import json from pathlib import Path +import shlex +import subprocess import sys @@ -17,11 +19,69 @@ from tools.elf_runtime_abi import fix_linux_runtime_abi from tools.nasm_direct_float_abi import append_direct_float_abi from tools.native_surface import public_exports +from tools.normalize_full_core_expr_stmt_assembly import ( + fix_expr_stmt_initializer_assembly, +) from tools.normalize_full_core_validation import main as normalize_full_runtime from tools.python_surface import PYTHON_MODULE_EXPORTS SOURCE = REPOSITORY_ROOT / "src" / "portapy" / "native_full_reference_entry.py" +COMPILER_WRAPPER = REPOSITORY_ROOT / "tools" / "run_full_core_asmpython.py" + + +def _prepare_full_runtime_sources() -> None: + """Apply the complete verified native normalization pipeline once.""" + normalize_full_runtime() + + +def _install_full_runtime_compiler() -> None: + """Route production compilation through the proven full-core CLI wrapper.""" + + def compile_python_source( + *, + target: str, + source: Path, + output: Path, + build_log: Path, + ) -> Path: + command = [ + sys.executable, + str(COMPILER_WRAPPER), + "build", + str(source), + "--target", + target, + "--type", + "library", + "--backend", + "legacy", + "--no-pyinbin-fallback", + "--keep-assembly", + "-o", + str(output), + ] + completed = subprocess.run( + command, + cwd=REPOSITORY_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + build_log.parent.mkdir(parents=True, exist_ok=True) + build_log.write_text(completed.stdout, encoding="utf-8") + + assembly = output.with_suffix(".asm") + if not assembly.is_file(): + rendered = shlex.join(command) + raise BuildFailure( + f"asmpython did not emit assembly (exit {completed.returncode}): " + f"{rendered}\n{completed.stdout}" + ) + return assembly + + base_build._compile_python_source = compile_python_source def _install_full_runtime_transforms() -> None: @@ -44,8 +104,15 @@ def transform( finally: base_build.append_float_abi = original_float + source = assembly.read_text(encoding="utf-8") + source, expr_stmt_count = fix_expr_stmt_initializer_assembly( + source, + target=target, + ) + assembly.write_text(source, encoding="utf-8") + print("FIXED NATIVE EXPRSTMT PARAMETER LOAD", expr_stmt_count) + if target == "linux": - source = assembly.read_text(encoding="utf-8") marker = "section .rodata" count = source.count(marker) if count < 1: @@ -73,7 +140,8 @@ def build_full_runtime( normalize: bool = True, ) -> dict[str, object]: if normalize: - normalize_full_runtime() + _prepare_full_runtime_sources() + _install_full_runtime_compiler() _install_full_runtime_transforms() metadata = base_build.build_native( diff --git a/tools/build_native_host_calls.py b/tools/build_native_host_calls.py index 82f94a39..7d8c4c04 100644 --- a/tools/build_native_host_calls.py +++ b/tools/build_native_host_calls.py @@ -76,6 +76,30 @@ def _compile_bridge_glue( _run(command, log=log) +def _linux_link_command( + *, + gcc: str, + objects: list[str], + version_script: Path, + output: Path, +) -> list[str]: + """Return a self-contained ELF shared-library link command. + + asmpython emits calls to ``pow`` for Python exponentiation. ``libportapy.so`` + must therefore declare its own libm dependency rather than relying on the + embedding process to have loaded libm globally already. + """ + return [ + gcc, + "-shared", + *objects, + f"-Wl,--version-script={version_script}", + "-lm", + "-o", + str(output), + ] + + def _upgrade_linked_artifact( *, target: str, @@ -179,14 +203,12 @@ def _upgrade_linked_artifact( linux_version_script(host_bridge=True, host_calls=True), encoding="utf-8", ) - command = [ - gcc, - "-shared", - *objects, - f"-Wl,--version-script={version_script}", - "-o", - str(output), - ] + command = _linux_link_command( + gcc=gcc, + objects=objects, + version_script=version_script, + output=output, + ) else: definition = work_dir / "portapy-host-calls.def" definition.write_text( diff --git a/tools/build_native_typed.py b/tools/build_native_typed.py index 9c852b38..825c7265 100644 --- a/tools/build_native_typed.py +++ b/tools/build_native_typed.py @@ -1,9 +1,9 @@ -"""Build PortaPy's current native interpreter entry. +"""Build PortaPy's stable full parser/VM runtime. -The historical filename remains the stable CI/release command. Default builds -now use the generated host-call entry, including scalar expressions, control -flow, positional functions, opaque host graphs, and synchronous callbacks. -Passing ``--source`` retains focused source-entry compiler probes. +The historical command remains the canonical CI and release entry. Default +builds emit the standalone parser, frontend, bytecode VM, host bridge, public +environment API, and complete stable value/container ABI. Passing ``--source`` +enables focused compiler probes without running the full-runtime pipeline. """ from __future__ import annotations @@ -18,7 +18,7 @@ sys.path.insert(0, str(REPOSITORY_ROOT)) from tools.build_native import BuildFailure, build_native -from tools.build_native_host_calls import main as build_host_call_entry +from tools.build_native_full_runtime import main as build_full_runtime_entry from tools.python_surface import PYTHON_MODULE_EXPORTS @@ -45,6 +45,8 @@ def _build_explicit_source(argv: list[str]) -> int: metadata["generated_function_entry"] = False metadata["generated_host_entry"] = False metadata["generated_host_call_entry"] = False + metadata["full_frontend_vm"] = False + metadata["standalone_parser"] = False metadata["python_module_exports"] = list(PYTHON_MODULE_EXPORTS) metadata["python_module_entry"] = "portapy" metadata_path = args.output.resolve().with_suffix(args.output.suffix + ".json") @@ -57,7 +59,7 @@ def main(argv: list[str] | None = None) -> int: arguments = list(sys.argv[1:] if argv is None else argv) if "--source" in arguments: return _build_explicit_source(arguments) - return build_host_call_entry(arguments) + return build_full_runtime_entry(arguments) if __name__ == "__main__": diff --git a/tools/combine_full_core_native_parser.py b/tools/combine_full_core_native_parser.py index bed39949..487a050e 100644 --- a/tools/combine_full_core_native_parser.py +++ b/tools/combine_full_core_native_parser.py @@ -86,6 +86,8 @@ def _prepare_runtime(module: ast.Module) -> list[ast.stmt]: def _rename_ast_arg_class(module: ast.Module) -> None: class_count = 0 call_count = 0 + existing_class_count = 0 + existing_call_count = 0 class _AnnotationRenamer(ast.NodeTransformer): def visit_Name(self, node: ast.Name) -> ast.AST: @@ -95,16 +97,18 @@ def visit_Name(self, node: ast.Name) -> ast.AST: renamer = _AnnotationRenamer() for node in ast.walk(module): - if isinstance(node, ast.ClassDef) and node.name == "arg": - node.name = "AstArg" - class_count += 1 - elif ( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id == "arg" - ): - node.func.id = "AstArg" - call_count += 1 + if isinstance(node, ast.ClassDef): + if node.name == "arg": + node.name = "AstArg" + class_count += 1 + elif node.name == "AstArg": + existing_class_count += 1 + elif isinstance(node, ast.Call) and isinstance(node.func, ast.Name): + if node.func.id == "arg": + node.func.id = "AstArg" + call_count += 1 + elif node.func.id == "AstArg": + existing_call_count += 1 if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): arguments = [ @@ -124,12 +128,27 @@ def visit_Name(self, node: ast.Name) -> ast.AST: elif isinstance(node, ast.AnnAssign): node.annotation = renamer.visit(node.annotation) - if class_count != 1 or call_count < 1: - raise RuntimeError( - "native AST arg rename failed: " - f"classes={class_count}, calls={call_count}" + if class_count == 1 and call_count >= 1: + print("RENAMED NATIVE AST ARG CLASS", class_count, call_count) + return + if ( + class_count == 0 + and call_count == 0 + and existing_class_count == 1 + and existing_call_count >= 1 + ): + print( + "PRESERVED RENAMED NATIVE AST ARG CLASS", + existing_class_count, + existing_call_count, ) - print("RENAMED NATIVE AST ARG CLASS", class_count, call_count) + return + raise RuntimeError( + "native AST arg rename failed: " + f"classes={class_count}, calls={call_count}, " + f"existing_classes={existing_class_count}, " + f"existing_calls={existing_call_count}" + ) def _prepare_bridge(module: ast.Module) -> list[ast.stmt]: diff --git a/tools/elf_pic.py b/tools/elf_pic.py index 34a6c8cc..2f48a9de 100644 --- a/tools/elf_pic.py +++ b/tools/elf_pic.py @@ -1,18 +1,27 @@ -"""Rewrite asmpython legacy NASM output into ELF shared-library-safe PIC. +"""Rewrite asmpython legacy NASM output for safe ELF shared libraries. -This is build/ABI glue only. It does not implement any PortaPy interpreter -semantics. The pass is intentionally narrow and fails on unsupported references -rather than producing a library with text relocations. +The legacy backend emits direct external references and follows Win64's more +permissive call-stack behavior. ELF shared libraries need PLT/GOT references, +and System V AMD64 requires ``rsp`` to be 16-byte aligned immediately before +``call``. This build-only pass fixes both properties and fails closed when the +control-flow analysis cannot prove a call's alignment. """ from __future__ import annotations import argparse -import re +from collections import defaultdict, deque +from dataclasses import dataclass from pathlib import Path +import re _EXTERN_DATA = {"stdin", "stdout", "stderr", "environ"} -_CALL_RE = re.compile(r"^(?P\s*)(?Pcall|jmp)\s+(?P[A-Za-z_.$?][\w.$?@]*)\s*$") +_NORETURN_CALLS = {"abort", "exit", "_exit", "longjmp", "_runtime_longjmp"} +_DIRECT_TRANSFER_RE = re.compile( + r"^(?P\s*)(?Pcall|jmp)\s+" + r"(?P[A-Za-z_.$?][\w.$?@]*)\s*$" +) +_CALL_RE = re.compile(r"^(?P\s*)call\s+(?P.+?)\s*$") _DATA_LOAD_RE = re.compile( r"^(?P\s*)mov\s+(?P[A-Za-z][A-Za-z0-9]*),\s*" r"\[(?:rel\s+)?(?P[A-Za-z_.$?][\w.$?@]*)\]\s*$" @@ -20,12 +29,36 @@ _EXTERNAL_MEMORY_RE = re.compile( r"\[(?:rel\s+)?(?P[A-Za-z_.$?][\w.$?@]*)[^\]]*\]" ) +_STACK_ADJUST_RE = re.compile( + r"^(?P\s*)(?Padd|sub)\s+rsp,\s*" + r"(?P0x[0-9A-Fa-f]+|\d+)\s*$" +) +_LABEL_RE = re.compile(r"^(?P