diff --git a/CHANGELOG.md b/CHANGELOG.md index c94815fd..cf220bde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.36.0] - 2026/MM/DD ### Fixed @@ -50,6 +50,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 setting `UV_BUILD_CONSTRAINT` alongside `PIP_CONSTRAINT` and `PIP_BUILD_CONSTRAINT`. [#389](https://github.com/pyodide/pyodide-build/pull/389) +### Changed + +- Fixed build-time scripts of unisolated packages (like `f2py` and + `numpy-config` from NumPy) not being available on `PATH` during builds. + [#21](https://github.com/pyodide/pyodide-build/pull/21) + ## [0.35.1] - 2026/06/13 ### Fixed @@ -222,12 +228,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added `pyodide clean recipes`, a CLI command that deletes build files for chosen packages or tags. -[#254](https://github.com/pyodide/pyodide-build/pull/254) + [#254](https://github.com/pyodide/pyodide-build/pull/254) ## [0.30.8] - 2025/10/22 - `pyodide config` now exposes `dist_dir` variable. -[#236](https://github.com/pyodide/pyodide-build/pull/236) + [#236](https://github.com/pyodide/pyodide-build/pull/236) - The CMake toolchain file for Pyodide now sets `CMAKE_SHARED_LINKER_FLAGS_INIT` and `CMAKE_MODULE_LINKER_FLAGS_INIT` and unset `CMAKE_SHARED_LINKER_FLAGS` to avoid conflicts with the user's settings. diff --git a/integration_tests/recipes/README.md b/integration_tests/recipes/README.md index c245e798..dc5d0c57 100644 --- a/integration_tests/recipes/README.md +++ b/integration_tests/recipes/README.md @@ -11,6 +11,7 @@ This directory contains a few curated recipes to test the build process of pyodi - `pydoc_data`: Unvendored cpython module - `boost-histogram``: Tests scikit-build-core and cmake build system. - `colorama`: Uses prebuilt wheel as source to check test file unvendoring capabilities. +- `numpy-scripts-example`: Tests that unisolated package scripts (say, `f2py` or `numpy-config` from NumPy as a build dependency) are available on PATH during builds. ### For maintainers diff --git a/integration_tests/recipes/numpy-scripts-example/meta.yaml b/integration_tests/recipes/numpy-scripts-example/meta.yaml new file mode 100644 index 00000000..77aef5b4 --- /dev/null +++ b/integration_tests/recipes/numpy-scripts-example/meta.yaml @@ -0,0 +1,23 @@ +package: + name: numpy-scripts-example + version: 0.1.0 + top-level: + - numpy_scripts_example + +source: + path: source + +requirements: + host: + - numpy + run: + - numpy + +about: + home: https://github.com/pyodide/pyodide-build + summary: Minimal integration test that verifies f2py and numpy-config are on PATH during the build + license: MIT + +extra: + recipe-maintainers: + - agriyakhetarpal diff --git a/integration_tests/recipes/numpy-scripts-example/source/meson.build b/integration_tests/recipes/numpy-scripts-example/source/meson.build new file mode 100644 index 00000000..818da621 --- /dev/null +++ b/integration_tests/recipes/numpy-scripts-example/source/meson.build @@ -0,0 +1,32 @@ +project('numpy-scripts-example', 'c', + version: '0.1.0', + license: 'MIT', + meson_version: '>= 1.3.0', +) + +py_mod = import('python') +py3 = py_mod.find_installation() +py3_dep = py3.dependency() + +# The only point of this recipe is to verify that NumPy's console scripts +# are discoverable on PATH when it is a build-time dependency. +f2py = find_program('f2py') +numpy_cflags = run_command('numpy-config', '--cflags', check: true).stdout().strip().split() + +incdir_numpy = run_command(py3, + ['-c', 'import numpy; print(numpy.get_include())'], + check: true, +).stdout().strip() + +py3.install_sources(['numpy_scripts_example/__init__.py'], + pure: false, + subdir: 'numpy_scripts_example') + +py3.extension_module('_add', + 'numpy_scripts_example/_add.c', + include_directories: incdir_numpy, + c_args: numpy_cflags, + dependencies: py3_dep, + install: true, + subdir: 'numpy_scripts_example', +) diff --git a/integration_tests/recipes/numpy-scripts-example/source/numpy_scripts_example/__init__.py b/integration_tests/recipes/numpy-scripts-example/source/numpy_scripts_example/__init__.py new file mode 100644 index 00000000..1be9f4ae --- /dev/null +++ b/integration_tests/recipes/numpy-scripts-example/source/numpy_scripts_example/__init__.py @@ -0,0 +1,3 @@ +from ._add import add + +__all__ = ["add"] diff --git a/integration_tests/recipes/numpy-scripts-example/source/numpy_scripts_example/_add.c b/integration_tests/recipes/numpy-scripts-example/source/numpy_scripts_example/_add.c new file mode 100644 index 00000000..9daa91e5 --- /dev/null +++ b/integration_tests/recipes/numpy-scripts-example/source/numpy_scripts_example/_add.c @@ -0,0 +1,28 @@ +#define PY_SSIZE_T_CLEAN +#include +#include + +static PyObject * +add(PyObject *self, PyObject *args) +{ + double a, b; + if (!PyArg_ParseTuple(args, "dd", &a, &b)) + return NULL; + return PyFloat_FromDouble(a + b); +} + +static PyMethodDef methods[] = { + {"add", add, METH_VARARGS, "Add two floats."}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef module = { + PyModuleDef_HEAD_INIT, "_add", NULL, -1, methods +}; + +PyMODINIT_FUNC +PyInit__add(void) +{ + import_array(); + return PyModule_Create(&module); +} diff --git a/integration_tests/recipes/numpy-scripts-example/source/pyproject.toml b/integration_tests/recipes/numpy-scripts-example/source/pyproject.toml new file mode 100644 index 00000000..70fc583e --- /dev/null +++ b/integration_tests/recipes/numpy-scripts-example/source/pyproject.toml @@ -0,0 +1,9 @@ +[build-system] +build-backend = "mesonpy" +requires = ["meson-python", "numpy"] + +[project] +name = "numpy-scripts-example" +version = "0.1.0" +requires-python = ">=3.9" +dependencies = ["numpy"] diff --git a/pyodide_build/build_env.py b/pyodide_build/build_env.py index 635e5f29..2f4afc2b 100644 --- a/pyodide_build/build_env.py +++ b/pyodide_build/build_env.py @@ -163,7 +163,6 @@ def get_build_environment_vars(pyodide_root: Path) -> dict[str, str]: "PYODIDE": "1", # This is the legacy environment variable used for the aforementioned purpose "PYODIDE_PACKAGE_ABI": "1", - "PYTHONPATH": env["HOSTSITEPACKAGES"], } ) @@ -220,29 +219,82 @@ def get_hostsitepackages() -> str: return get_build_flag("HOSTSITEPACKAGES") +# TODO: Remove this function (and use remote package index) +# https://github.com/pyodide/pyodide-build/issues/43 @functools.cache -def get_unisolated_packages() -> list[str]: - # TODO: Remove this function (and use remote package index) - # https://github.com/pyodide/pyodide-build/issues/43 +def get_unisolated_packages() -> dict[str, str]: + """ + Get a map of unisolated packages. + + Unisolated packages are packages that are used during the build process + and have some platform-specific files. When these packages are used + during the build process, their platform-specific files are replaced with + WASM-compatible versions to build the package correctly. + + Returns + ------- + A dictionary of package names and versions. + """ PYODIDE_ROOT = get_pyodide_root() - unisolated_file = PYODIDE_ROOT / "unisolated.txt" - if unisolated_file.exists(): - # in xbuild env, read from file - unisolated_packages = unisolated_file.read_text().splitlines() + unisolated_packages: dict[str, str] = {} + if in_xbuildenv(): + unisolated_packages_file = PYODIDE_ROOT / ".." / "requirements.txt" + + if not unisolated_packages_file.exists(): + raise FileNotFoundError( + f"Expected {unisolated_packages_file} to exist in the xbuildenv. " + "The xbuildenv archive may be corrupt or from an incompatible version." + ) + for line in unisolated_packages_file.read_text().splitlines(): + line = line.strip() + # The xbuildenv requirements.txt is machine-generated and always + # uses pinned name==version entries, but we skip blank lines, + # comments, and any non-pinned specs just in case. Shouldn't + # really happen though. + if not line or line.startswith("#") or "==" not in line: + continue + name, version = line.split("==", 1) + unisolated_packages[name] = version else: from pyodide_build.recipe.loader import load_all_recipes - unisolated_packages = [] recipe_dir = PYODIDE_ROOT / "packages" recipes = load_all_recipes(recipe_dir) for name, config in recipes.items(): if config.build.cross_build_env: - unisolated_packages.append(name) + unisolated_packages[name] = config.package.version return unisolated_packages +def get_cross_build_files_dir(package_name: str) -> Path: + """ + Get the directory containing an unisolated package's cross-build files + (such as headers, .a libs, .pxd files, and so on). + + Parameters + ---------- + package_name + The name of the package + + Returns + ------- + The directory containing the package's cross-build files, relative to + which they should be laid out in site-packages. The directory may not + exist if the package has no cross-build files. + """ + PYODIDE_ROOT = get_pyodide_root() + + # TODO: unify libdir for in-tree and out-of-tree builds + if in_xbuildenv(): + libdir = PYODIDE_ROOT / ".." / "site-packages-extras" + else: + libdir = Path(get_hostsitepackages()) + + return libdir / package_name + + def platform() -> str: emscripten_version = get_build_flag("PYODIDE_EMSCRIPTEN_VERSION") version = emscripten_version.replace(".", "_") diff --git a/pyodide_build/pypabuild.py b/pyodide_build/pypabuild.py index f14a8836..8eb6807e 100644 --- a/pyodide_build/pypabuild.py +++ b/pyodide_build/pypabuild.py @@ -4,22 +4,23 @@ import subprocess as sp import sys import traceback +import warnings from collections.abc import Callable, Iterator, Mapping, Sequence from contextlib import contextmanager -from itertools import chain from pathlib import Path from typing import Literal, cast from build import BuildBackendException, ConfigSettingsType, ProjectBuilder from build.env import DefaultIsolatedEnv from packaging.requirements import Requirement +from packaging.utils import canonicalize_name from pyodide_build import _f2c_fixes, common, pywasmcross, uv_helper from pyodide_build.build_env import ( get_build_flag, + get_cross_build_files_dir, get_current_xbuildenv_manager, get_host_build_flag, - get_hostsitepackages, get_pyversion, get_unisolated_packages, in_xbuildenv, @@ -30,6 +31,7 @@ _configure_build_verbosity, _DefaultIsolatedEnv, _error, + _find_executable_and_scripts, _handle_build_error, _styles, ) @@ -96,11 +98,11 @@ def _runner(cmd, cwd=None, extra_environ=None): return _runner -def symlink_unisolated_packages( - env: DefaultIsolatedEnv, reqs: set[str] | None = None -) -> None: - from pyodide_build.build_env import get_build_flag, get_unisolated_packages - +def _copy_sysconfigdata_to_isolated_env(env: DefaultIsolatedEnv) -> None: + """ + Copy the sysconfigdata module into the isolated build environment's + site-packages so that builds can pick up Pyodide's build configuration. + """ pyversion = get_pyversion() site_packages_path = f"lib/{pyversion}/site-packages" env_site_packages = Path(env.path) / site_packages_path @@ -112,29 +114,106 @@ def symlink_unisolated_packages( env_site_packages.mkdir(parents=True, exist_ok=True) shutil.copy(sysconfigdata_path, env_site_packages) - host_site_packages = Path(get_hostsitepackages()) - unisolated_packages = set(get_unisolated_packages()) - required = {Requirement(req).name.lower() for req in (reqs or set())} - needs_cross_build_install = bool(unisolated_packages & required) +def _replace_unisolated_packages( + reqs: set[str], unisolated_packages: dict[str, str] +) -> tuple[set[str], set[str]]: + """ + Replace unisolated packages with the correct version. - if in_xbuildenv() and needs_cross_build_install: - get_current_xbuildenv_manager().ensure_cross_build_packages_installed() + Parameters + ---------- + reqs + The set of requirements to filter. + unisolated_packages + The dictionary of unisolated packages [name: version]. + + Returns + ------- + A tuple of (the filtered set of requirements, the set of unisolated requirements) + """ + canonical_unisolated = { + canonicalize_name(name): (name, version) + for name, version in unisolated_packages.items() + } + + new_reqs = reqs.copy() + unisolated: set[str] = set() + for reqstr in reqs: + req = Requirement(reqstr) + # Evaluate the PEP 508 marker to see if the requirement + # is applicable in the current environment or not. + if req.marker and not req.marker.evaluate(): + continue + match = canonical_unisolated.get(canonicalize_name(req.name)) + if match is None: + continue + name, version = match + # TODO: find a better way to handle this case + if not req.specifier.contains(version): + warnings.warn( + f"Found build dependency {req} but the only supported " + f"cross-build version is {name}=={version}; " + f"using {name}=={version} instead.", + stacklevel=2, + ) + new_reqs.discard(reqstr) + new_reqs.add(f"{name}=={version}") + unisolated.add(name) + return new_reqs, unisolated - for name in get_unisolated_packages(): - for path in chain( - host_site_packages.glob(f"{name}*"), host_site_packages.glob(f"_{name}*") - ): - (env_site_packages / path.name).unlink(missing_ok=True) - (env_site_packages / path.name).symlink_to(path) + +def _install_cross_build_files(venv_path: str, unisolated: set[str]) -> None: + """ + Install the cross build files (headers, .a libs, .pxd files) to the + isolated environment's site packages. + + Parameters + ---------- + venv_path + The path to the isolated environment. + + unisolated + The set of unisolated packages. + """ + if not unisolated: + return + _, _, purelib = _find_executable_and_scripts(venv_path) + sitepackagesdir = Path(purelib) + for name in unisolated: + package_dir = get_cross_build_files_dir(name) + if not package_dir.is_dir(): + # Not every unisolated package has cross-build files. The package + # may only need to be pinned to the cross-build version (for its + # console scripts, for instance) without any file overlay. + continue + shutil.copytree(package_dir, sitepackagesdir / name, dirs_exist_ok=True) def remove_avoided_requirements( requires: set[str], avoided_requirements: set[str] | list[str] ) -> set[str]: + """ + Remove requirements that are in the list of avoided requirements. + + Parameters + ---------- + requires + The set of requirements to filter. + avoided_requirements + The set of requirements to avoid. + + Returns + ------- + The filtered set of requirements. + """ for reqstr in list(requires): req = Requirement(reqstr) + # Evaluate the PEP 508 marker to see if the requirement + # is applicable in the current environment or not. + if req.marker and not req.marker.evaluate(): + continue for avoid_name in set(avoided_requirements): if avoid_name == req.name.lower(): requires.remove(reqstr) @@ -147,16 +226,32 @@ def install_reqs( IGNORED_BUILD_REQUIREMENTS = [ pkg.strip() for pkg in get_host_build_flag("IGNORED_BUILD_REQUIREMENTS").split() ] + reqs, unisolated = _replace_unisolated_packages(reqs, get_unisolated_packages()) + reqs = remove_avoided_requirements(reqs, IGNORED_BUILD_REQUIREMENTS) + + if in_xbuildenv() and unisolated: + get_current_xbuildenv_manager().ensure_cross_build_packages_installed() + # propagate PIP config from build_env to current environment with common.replace_env( os.environ | {k: v for k, v in build_env.items() if k.startswith("PIP")} ): - env.install( - remove_avoided_requirements( - reqs, - get_unisolated_packages() + IGNORED_BUILD_REQUIREMENTS, - ) - ) + env.install(reqs) + + _install_cross_build_files(env.path, unisolated) + + +# So far among all packages in pyodide-recipes, only NumPy ships +# a .pc file, but I don't want to hardcode that here as such +def _get_unisolated_pkgconfig_dirs(venv_path: str) -> list[str]: + """ + Find directories containing .pc files shipped by packages installed in the + isolated build environment. These need to be added to PKG_CONFIG_LIBDIR so + that meson can discover unisolated packages (like numpy) via pkg-config + during cross-compilation. + """ + _, _, purelib = _find_executable_and_scripts(venv_path) + return list({str(pc.parent) for pc in Path(purelib).rglob("*.pc") if pc.is_file()}) def _build_in_isolated_env( @@ -181,7 +276,7 @@ def _build_in_isolated_env( ) # first install the build dependencies - symlink_unisolated_packages(env, builder.build_system_requires) + _copy_sysconfigdata_to_isolated_env(env) install_reqs(build_env, env, builder.build_system_requires) build_reqs: set[str] | None = None try: @@ -208,6 +303,14 @@ def _build_in_isolated_env( install_reqs(build_env, env, build_reqs) + pkgconfig_dirs = _get_unisolated_pkgconfig_dirs(env.path) + if pkgconfig_dirs: + build_env = dict(build_env) + existing = build_env.get("PKG_CONFIG_LIBDIR", "") + build_env["PKG_CONFIG_LIBDIR"] = ":".join( + [existing, *pkgconfig_dirs] if existing else pkgconfig_dirs + ) + with common.replace_env(build_env): return builder.build( distribution, diff --git a/pyodide_build/tests/_test_xbuildenv/xbuildenv-test.tar.gz b/pyodide_build/tests/_test_xbuildenv/xbuildenv-test.tar.gz index a20ea97a..c2909812 100644 Binary files a/pyodide_build/tests/_test_xbuildenv/xbuildenv-test.tar.gz and b/pyodide_build/tests/_test_xbuildenv/xbuildenv-test.tar.gz differ diff --git a/pyodide_build/tests/conftest.py b/pyodide_build/tests/conftest.py index 133fc814..07515f2c 100644 --- a/pyodide_build/tests/conftest.py +++ b/pyodide_build/tests/conftest.py @@ -87,7 +87,8 @@ def mock_get_host_build_flag(flag_name): manager = CrossBuildEnvManager(default_xbuildenv_path()) manager.install( - version=None, url=dummy_xbuildenv_url, skip_install_cross_build_packages=True + version=None, + url=dummy_xbuildenv_url, ) cur_dir = os.getcwd() diff --git a/pyodide_build/tests/test_build_env.py b/pyodide_build/tests/test_build_env.py index 51923a23..c0d27253 100644 --- a/pyodide_build/tests/test_build_env.py +++ b/pyodide_build/tests/test_build_env.py @@ -59,6 +59,53 @@ def test_get_pyodide_root_pyodide_root_already_set( def test_in_xbuildenv(self, dummy_xbuildenv, reset_env_vars, reset_cache): assert build_env.in_xbuildenv() + def test_get_unisolated_packages( + self, dummy_xbuildenv, reset_env_vars, reset_cache + ): + manager = CrossBuildEnvManager(dummy_xbuildenv / common.xbuildenv_dirname()) + requirements_file = manager.pyodide_root / ".." / "requirements.txt" + + expected = {} + for line in requirements_file.read_text().splitlines(): + name, version = line.strip().split("==", 1) + expected[name] = version + + assert expected + assert build_env.get_unisolated_packages() == expected + + def test_get_unisolated_packages_no_requirements_file( + self, dummy_xbuildenv, reset_env_vars, reset_cache + ): + manager = CrossBuildEnvManager(dummy_xbuildenv / common.xbuildenv_dirname()) + requirements_file = manager.pyodide_root / ".." / "requirements.txt" + requirements_file.unlink() + + with pytest.raises(FileNotFoundError, match="Expected .* to exist"): + build_env.get_unisolated_packages() + + def test_get_cross_build_files_dir( + self, dummy_xbuildenv, reset_env_vars, reset_cache + ): + manager = CrossBuildEnvManager(dummy_xbuildenv / common.xbuildenv_dirname()) + site_packages_extras = manager.pyodide_root / ".." / "site-packages-extras" + + # Check every package that actually has a directory in site-packages-extras + found_any = False + for subdir in site_packages_extras.iterdir(): + if not subdir.is_dir(): + continue + found_any = True + package_dir = build_env.get_cross_build_files_dir(subdir.name) + assert package_dir == subdir + assert any(package_dir.rglob("*")) + assert found_any + + def test_get_cross_build_files_dir_missing_package( + self, dummy_xbuildenv, reset_env_vars, reset_cache + ): + package_dir = build_env.get_cross_build_files_dir("no-such-package") + assert not package_dir.exists() + def test_get_build_environment_vars( self, dummy_xbuildenv, reset_env_vars, reset_cache ): @@ -66,7 +113,7 @@ def test_get_build_environment_vars( build_vars = build_env.get_build_environment_vars(manager.pyodide_root) # extra variables that does not come from config files. - extra_vars = {"PYODIDE", "PYODIDE_PACKAGE_ABI", "PYTHONPATH"} + extra_vars = {"PYODIDE", "PYODIDE_PACKAGE_ABI"} all_keys = set(BUILD_KEY_TO_VAR.values()) | extra_vars for var in build_vars: diff --git a/pyodide_build/tests/test_pypabuild.py b/pyodide_build/tests/test_pypabuild.py index 435ecae4..820df85d 100644 --- a/pyodide_build/tests/test_pypabuild.py +++ b/pyodide_build/tests/test_pypabuild.py @@ -30,7 +30,51 @@ def test_remove_avoided_requirements(): ) == {"baz"} -def test_install_reqs(tmp_path, dummy_xbuildenv): +def test_replace_unisolated_packages(): + requires = {"foo", "bar<1.0", "baz==1.0", "qux"} + unisolated = { + "foo": "2.0", + "bar": "0.5", + "baz": "1.0", + } + + new_requires, replaced = pypabuild._replace_unisolated_packages( + requires, unisolated + ) + assert new_requires == {"foo==2.0", "bar==0.5", "baz==1.0", "qux"} + assert replaced == {"foo", "bar", "baz"} + + +def test_replace_unisolated_packages_normalizes_names(): + requires = {"NumPy>=1.20", "Ruamel-YAML"} + unisolated = { + "numpy": "2.0.3", + "ruamel.yaml": "0.18.6", + } + + new_requires, replaced = pypabuild._replace_unisolated_packages( + requires, unisolated + ) + assert new_requires == {"numpy==2.0.3", "ruamel.yaml==0.18.6"} + assert replaced == {"numpy", "ruamel.yaml"} + + +def test_replace_unisolated_packages_version_mismatch(): + requires = {"baz==1.0"} + unisolated = { + "baz": "1.1", + } + + with pytest.warns(UserWarning, match=r"cross-build version is baz==1\.1"): + new_requires, replaced = pypabuild._replace_unisolated_packages( + requires, unisolated + ) + assert new_requires == {"baz==1.1"} + assert replaced == {"baz"} + + +def test_install_reqs(tmp_path, dummy_xbuildenv, monkeypatch): + monkeypatch.setattr(pypabuild, "_install_cross_build_files", lambda *a, **kw: None) env = MockIsolatedEnv(tmp_path) reqs = {"foo", "bar", "baz"} @@ -110,28 +154,103 @@ def test_get_build_env(tmp_path, dummy_xbuildenv): assert "exports" in wasmcross_args -def test_symlink_unisolated_packages_triggers_lazy_install( - tmp_path, dummy_xbuildenv, monkeypatch, reset_env_vars, reset_cache -): +def test_install_reqs_triggers_lazy_install(tmp_path, monkeypatch): called = {"count": 0} - def _ensure(self): - called["count"] += 1 + class DummyManager: + def ensure_cross_build_packages_installed(self): + called["count"] += 1 + + monkeypatch.setattr(pypabuild, "in_xbuildenv", lambda: True) + monkeypatch.setattr(pypabuild, "get_current_xbuildenv_manager", DummyManager) + monkeypatch.setattr(pypabuild, "get_unisolated_packages", lambda: {"numpy": "1.0"}) + monkeypatch.setattr(pypabuild, "_install_cross_build_files", lambda *a, **kw: None) + + env = MockIsolatedEnv(tmp_path) + pypabuild.install_reqs({}, env, {"numpy>=1.0"}) + + assert called["count"] == 1 + + +def test_install_reqs_skips_lazy_install_when_not_unisolated(tmp_path, monkeypatch): + called = {"count": 0} + + class DummyManager: + def ensure_cross_build_packages_installed(self): + called["count"] += 1 + + monkeypatch.setattr(pypabuild, "in_xbuildenv", lambda: True) + monkeypatch.setattr(pypabuild, "get_current_xbuildenv_manager", DummyManager) + monkeypatch.setattr(pypabuild, "get_unisolated_packages", lambda: {"numpy": "1.0"}) + monkeypatch.setattr(pypabuild, "_install_cross_build_files", lambda *a, **kw: None) + + env = MockIsolatedEnv(tmp_path) + pypabuild.install_reqs({}, env, {"foo>=1.0"}) + + assert called["count"] == 0 + + +def test_install_cross_build_files(tmp_path, monkeypatch): + purelib = tmp_path / "venv" / "lib" / "site-packages" + purelib.mkdir(parents=True) + + extras = tmp_path / "site-packages-extras" + numpy_header = extras / "numpy" / "_core" / "include" / "numpy" / "ndarrayobject.h" + numpy_header.parent.mkdir(parents=True) + numpy_header.write_text("// header") + scipy_pxd = extras / "scipy" / "linalg" / "cython_blas.pxd" + scipy_pxd.parent.mkdir(parents=True) + scipy_pxd.write_text("# pxd") monkeypatch.setattr( - "pyodide_build.xbuildenv.CrossBuildEnvManager.ensure_cross_build_packages_installed", - _ensure, + pypabuild, + "_find_executable_and_scripts", + lambda venv_path: ("python", "scripts", str(purelib)), ) monkeypatch.setattr( - "pyodide_build.build_env.get_unisolated_packages", - lambda: ["numpy"], + pypabuild, "get_cross_build_files_dir", lambda name: extras / name ) - class DummyEnv: - path = str(tmp_path / "venv") + pypabuild._install_cross_build_files(str(tmp_path / "venv"), {"numpy", "scipy"}) - pypabuild.symlink_unisolated_packages(DummyEnv(), reqs={"numpy>=1.0"}) - assert called["count"] == 1 + assert ( + purelib / "numpy" / "_core" / "include" / "numpy" / "ndarrayobject.h" + ).read_text() == "// header" + assert (purelib / "scipy" / "linalg" / "cython_blas.pxd").read_text() == "# pxd" + + +def test_install_cross_build_files_skips_packages_without_cross_build_files( + tmp_path, monkeypatch +): + purelib = tmp_path / "venv" / "lib" / "site-packages" + purelib.mkdir(parents=True) + + monkeypatch.setattr( + pypabuild, + "_find_executable_and_scripts", + lambda venv_path: ("python", "scripts", str(purelib)), + ) + monkeypatch.setattr( + pypabuild, + "get_cross_build_files_dir", + lambda name: tmp_path / "does-not-exist" / name, + ) + + pypabuild._install_cross_build_files(str(tmp_path / "venv"), {"some-package"}) + + assert list(purelib.iterdir()) == [] + + +def test_install_cross_build_files_skips_when_no_unisolated_packages( + tmp_path, monkeypatch +): + def _unexpected_call(*args, **kwargs): + raise AssertionError("should not be called when there are no unisolated reqs") + + monkeypatch.setattr(pypabuild, "_find_executable_and_scripts", _unexpected_call) + monkeypatch.setattr(pypabuild, "get_cross_build_files_dir", _unexpected_call) + + pypabuild._install_cross_build_files(str(tmp_path / "venv"), set()) def _make_cpe( diff --git a/pyodide_build/vendor/LICENSE b/pyodide_build/vendor/LICENSE new file mode 100644 index 00000000..c3713cdc --- /dev/null +++ b/pyodide_build/vendor/LICENSE @@ -0,0 +1,20 @@ +Copyright © 2019 Filipe Laíns + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice (including the next +paragraph) shall be included in all copies or substantial portions of the +Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/pyodide_build/vendor/_pypabuild.py b/pyodide_build/vendor/_pypabuild.py index c3b68bbb..7ba559d4 100644 --- a/pyodide_build/vendor/_pypabuild.py +++ b/pyodide_build/vendor/_pypabuild.py @@ -1,30 +1,14 @@ # This file contains private functions taken from pypa/build. - -# Copyright © 2019 Filipe Laíns - -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice (including the next -# paragraph) shall be included in all copies or substantial portions of the -# Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. +# See license at LICENSE next to this file and as defined in +# pyproject.toml. For built distributions of pyodide-build, +# you may obtain a copy of the license in the +# .dist-info/licenses directory. import contextlib import contextvars import os import subprocess import sys +import sysconfig import traceback import warnings from collections.abc import Callable, Iterator @@ -151,3 +135,52 @@ def _handle_build_error() -> Iterator[None]: _log_subprocess_output(cpe) _error(str(e)) + + +# Vendored from pypa/build v1.5.0. See source at: +# https://github.com/pypa/build/blob/615d04cfc52ac3c1592a463f0afe484fee1cc368/src/build/env.py#L461-L501 +def _find_executable_and_scripts(path: str) -> tuple[str, str, str]: + """ + Detect the Python executable and script folder of a virtual environment. + + :param path: The location of the virtual environment + :return: The Python executable, script folder, and purelib folder + """ + config_vars = ( + sysconfig.get_config_vars().copy() + ) # globally cached, copy before altering it + config_vars["base"] = path + scheme_names = sysconfig.get_scheme_names() + if "venv" in scheme_names: + # Python distributors with custom default installation scheme can set a + # scheme that can't be used to expand the paths in a venv. + # This can happen if build itself is not installed in a venv. + # The distributors are encouraged to set a "venv" scheme to be used for this. + # See https://bugs.python.org/issue45413 + # and https://github.com/pypa/virtualenv/issues/2208 + paths = sysconfig.get_paths(scheme="venv", vars=config_vars) # pragma: no cover + elif "posix_local" in scheme_names: + # The Python that ships on Debian/Ubuntu varies the default scheme to + # install to /usr/local + # But it does not (yet) set the "venv" scheme. + # If we're the Debian "posix_local" scheme is available, but "venv" + # is not, we use "posix_prefix" instead which is venv-compatible there. + paths = sysconfig.get_paths(scheme="posix_prefix", vars=config_vars) + elif "osx_framework_library" in scheme_names: + # The Python that ships with the macOS developer tools varies the + # default scheme depending on whether the ``sys.prefix`` is part of a framework. + # But it does not (yet) set the "venv" scheme. + # If the Apple-custom "osx_framework_library" scheme is available but "venv" + # is not, we use "posix_prefix" instead which is venv-compatible there. + paths = sysconfig.get_paths(scheme="posix_prefix", vars=config_vars) + else: + paths = sysconfig.get_paths(vars=config_vars) + + executable = os.path.join( + paths["scripts"], "python.exe" if os.name == "nt" else "python" + ) + if not os.path.exists(executable): + msg = f"Virtual environment creation failed, executable {executable} missing" + raise RuntimeError(msg) + + return executable, paths["scripts"], paths["purelib"] diff --git a/pyproject.toml b/pyproject.toml index add5d6ff..33e8da07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ classifiers = [ "Operating System :: OS Independent", ] license = "MPL-2.0" -license-files = ["LICENSE"] +license-files = ["LICENSE", "pyodide_build/vendor/LICENSE"] requires-python = ">=3.12" dependencies = [ "build>=1.4,<1.6,!=1.4.4", # keep in sync with uv extra