From 29468da967114ff821fb328b259d3ed7ec5bfb27 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 10 Apr 2025 16:16:22 +0530 Subject: [PATCH 01/12] Add `AVOIDED_BUILD_REQUIREMENTS` config variable --- pyodide_build/config.py | 5 ++++- pyodide_build/pypabuild.py | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pyodide_build/config.py b/pyodide_build/config.py index 0310e0e3..0692e59c 100644 --- a/pyodide_build/config.py +++ b/pyodide_build/config.py @@ -11,6 +11,7 @@ search_pyproject_toml, ) from pyodide_build.logger import logger +from pyodide_build.pypabuild import AVOIDED_REQUIREMENTS class ConfigManager: @@ -199,6 +200,7 @@ def _get_make_environment_vars(self) -> Mapping[str, str]: "build_dependency_index_url": "BUILD_DEPENDENCY_INDEX_URL", "default_cross_build_env_url": "DEFAULT_CROSS_BUILD_ENV_URL", "xbuildenv_path": "PYODIDE_XBUILDENV_PATH", + "avoided_build_requirements": "AVOIDED_BUILD_REQUIREMENTS", # maintainer only "_f2c_fixes_wrapper": "_F2C_FIXES_WRAPPER", } @@ -218,6 +220,7 @@ def _get_make_environment_vars(self) -> Mapping[str, str]: "build_dependency_index_url", "default_cross_build_env_url", "xbuildenv_path", + "avoided_build_requirements", # maintainer only "_f2c_fixes_wrapper", } @@ -235,11 +238,11 @@ def _get_make_environment_vars(self) -> Mapping[str, str]: "rust_toolchain": "nightly-2025-02-01", "rust_emscripten_target_url": "", # Other configuration - "pyodide_jobs": "1", "skip_emscripten_version_check": "0", "build_dependency_index_url": "https://pypi.anaconda.org/pyodide/simple", "default_cross_build_env_url": "", "xbuildenv_path": "", + "avoided_build_requirements": " ".join(AVOIDED_REQUIREMENTS), # maintainer only "_f2c_fixes_wrapper": "", } diff --git a/pyodide_build/pypabuild.py b/pyodide_build/pypabuild.py index 17105057..7af18011 100644 --- a/pyodide_build/pypabuild.py +++ b/pyodide_build/pypabuild.py @@ -17,6 +17,7 @@ from pyodide_build import _f2c_fixes, common, pywasmcross, uv_helper from pyodide_build.build_env import ( get_build_flag, + get_host_build_flag, get_hostsitepackages, get_pyversion, get_unisolated_packages, @@ -37,6 +38,11 @@ "oldest-supported-numpy", ] +AVOIDED_REQUIREMENTS: list[str] = [ + "patchelf", + "oldest-supported-numpy", +] + get_host_build_flag("AVOIDED_BUILD_REQUIREMENTS").split() + # corresponding env variables for symlinks SYMLINK_ENV_VARS = { "cc": "CC", From 13929c00c6e68a0ff4c9383b614e3388b8aac9dc Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 10 Apr 2025 16:18:06 +0530 Subject: [PATCH 02/12] Wrap `pyodide config list` outputs in strings --- pyodide_build/cli/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyodide_build/cli/config.py b/pyodide_build/cli/config.py index 86bec1bd..0972b82f 100644 --- a/pyodide_build/cli/config.py +++ b/pyodide_build/cli/config.py @@ -32,7 +32,7 @@ def list_config(): configs = _get_configs() for k, v in configs.items(): - typer.echo(f"{k}={v}") + typer.echo(f'{k}="{v}"') @app.command("get") From 4442d41b65b7dfb7d96f23fab5e24fd6748a07f8 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 10 Apr 2025 16:18:14 +0530 Subject: [PATCH 03/12] Add a CHANGELOG entry for #187 --- CHANGELOG.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86629301..bbeb7ed6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,16 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `pyodide config` exposes new variables: `pyodide_root`, `pyodide_abi_version`, and `python_include_dir` - [#186](https://github.com/pyodide/pyodide-build/pull/186) +- `pyodide config` exposes new variables: `pyodide_root`, `pyodide_abi_version`, and `python_include_dir`, + and `ignored_build_requirements`. + [#186](https://github.com/pyodide/pyodide-build/pull/186) and [#187](https://github.com/pyodide/pyodide-build/pull/187) ## [0.30.0] - 2025/04/08 ### Added -- Added basic support for uv. `uv tool install pyodide-cli --with pyodide-build`, or - `uvx --from pyodide-cli --with pyodide-build pyodide --help`, or using `pyodide-build` - in `uv`-managed virtual environments will now work. +- Added basic support for uv. `uv tool install pyodide-cli --with pyodide-build`, or `uvx --from pyodide-cli --with pyodide-build pyodide --help`, or using `pyodide-build` in `uv`-managed virtual environments will now work. [#132](https://github.com/pyodide/pyodide-build/pull/132) - `pyodide build` now takes an additional `--xbuildenv-path` argument and corresponding From 82c1849854a94dfd8d86ccc04f3e4c511596bab8 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 10 Apr 2025 17:51:01 +0530 Subject: [PATCH 04/12] Add BASE_IGNORED_REQUIREMENTS and rename --- pyodide_build/config.py | 9 +++++---- pyodide_build/constants.py | 8 ++++++++ pyodide_build/pypabuild.py | 20 +++++++------------- pyodide_build/tests/test_pypabuild.py | 4 ++-- 4 files changed, 22 insertions(+), 19 deletions(-) create mode 100644 pyodide_build/constants.py diff --git a/pyodide_build/config.py b/pyodide_build/config.py index 0692e59c..ea08c8c8 100644 --- a/pyodide_build/config.py +++ b/pyodide_build/config.py @@ -10,8 +10,8 @@ exit_with_stdio, search_pyproject_toml, ) +from pyodide_build.constants import BASE_IGNORED_REQUIREMENTS from pyodide_build.logger import logger -from pyodide_build.pypabuild import AVOIDED_REQUIREMENTS class ConfigManager: @@ -200,7 +200,7 @@ def _get_make_environment_vars(self) -> Mapping[str, str]: "build_dependency_index_url": "BUILD_DEPENDENCY_INDEX_URL", "default_cross_build_env_url": "DEFAULT_CROSS_BUILD_ENV_URL", "xbuildenv_path": "PYODIDE_XBUILDENV_PATH", - "avoided_build_requirements": "AVOIDED_BUILD_REQUIREMENTS", + "ignored_build_requirements": "IGNORED_BUILD_REQUIREMENTS", # maintainer only "_f2c_fixes_wrapper": "_F2C_FIXES_WRAPPER", } @@ -220,7 +220,7 @@ def _get_make_environment_vars(self) -> Mapping[str, str]: "build_dependency_index_url", "default_cross_build_env_url", "xbuildenv_path", - "avoided_build_requirements", + "ignored_build_requirements", # maintainer only "_f2c_fixes_wrapper", } @@ -242,7 +242,8 @@ def _get_make_environment_vars(self) -> Mapping[str, str]: "build_dependency_index_url": "https://pypi.anaconda.org/pyodide/simple", "default_cross_build_env_url": "", "xbuildenv_path": "", - "avoided_build_requirements": " ".join(AVOIDED_REQUIREMENTS), + # A list of PEP508 build-time requirements to be ignored when building a wheel + "ignored_build_requirements": " ".join(BASE_IGNORED_REQUIREMENTS), # maintainer only "_f2c_fixes_wrapper": "", } diff --git a/pyodide_build/constants.py b/pyodide_build/constants.py new file mode 100644 index 00000000..53d275d3 --- /dev/null +++ b/pyodide_build/constants.py @@ -0,0 +1,8 @@ +# Some reusable constants that are used at various sources in our code. +# TODO: collect and add more constants here. + +BASE_IGNORED_REQUIREMENTS: list[str] = [ + # mesonpy installs patchelf in linux platform but we don't want it. + "patchelf", + "oldest-supported-numpy", +] diff --git a/pyodide_build/pypabuild.py b/pyodide_build/pypabuild.py index 7af18011..c8823c18 100644 --- a/pyodide_build/pypabuild.py +++ b/pyodide_build/pypabuild.py @@ -32,17 +32,6 @@ _ProjectBuilder, ) -AVOIDED_REQUIREMENTS = [ - # mesonpy installs patchelf in linux platform but we don't want it. - "patchelf", - "oldest-supported-numpy", -] - -AVOIDED_REQUIREMENTS: list[str] = [ - "patchelf", - "oldest-supported-numpy", -] + get_host_build_flag("AVOIDED_BUILD_REQUIREMENTS").split() - # corresponding env variables for symlinks SYMLINK_ENV_VARS = { "cc": "CC", @@ -103,6 +92,8 @@ def _runner(cmd, cwd=None, extra_environ=None): def symlink_unisolated_packages(env: DefaultIsolatedEnv) -> None: + from pyodide_build.build_env import get_build_flag, get_unisolated_packages + pyversion = get_pyversion() site_packages_path = f"lib/{pyversion}/site-packages" env_site_packages = Path(env.path) / site_packages_path @@ -137,6 +128,9 @@ def remove_avoided_requirements( def install_reqs( build_env: Mapping[str, str], env: DefaultIsolatedEnv, reqs: set[str] ) -> None: + IGNORED_BUILD_REQUIREMENTS = get_host_build_flag( + "IGNORED_BUILD_REQUIREMENTS" + ).split() # propagate PIP config from build_env to current environment with common.replace_env( os.environ | {k: v for k, v in build_env.items() if k.startswith("PIP")} @@ -144,7 +138,7 @@ def install_reqs( env.install( remove_avoided_requirements( reqs, - get_unisolated_packages() + AVOIDED_REQUIREMENTS, + get_unisolated_packages() + IGNORED_BUILD_REQUIREMENTS, ) ) @@ -262,7 +256,6 @@ def make_command_wrapper_symlinks(symlink_dir: Path) -> dict[str, str]: ------- The dictionary of compiler environment variables that points to the symlinks. """ - # For maintainers: # - you can set "_f2c_fixes_wrapper" variable in pyproject.toml # in order to change the script to use when cross-compiling @@ -321,6 +314,7 @@ def get_build_env( Returns a dict of environment variables that should be used when building a package with pypa/build. """ + from pyodide_build.build_env import get_build_flag kwargs = { "pkgname": pkgname, diff --git a/pyodide_build/tests/test_pypabuild.py b/pyodide_build/tests/test_pypabuild.py index 5c6af469..82f2a1eb 100644 --- a/pyodide_build/tests/test_pypabuild.py +++ b/pyodide_build/tests/test_pypabuild.py @@ -27,8 +27,8 @@ def test_install_reqs(tmp_path, dummy_xbuildenv): for req in reqs: assert req in env.installed - pypabuild.install_reqs({}, env, set(pypabuild.AVOIDED_REQUIREMENTS)) # type: ignore[arg-type] - for req in pypabuild.AVOIDED_REQUIREMENTS: + pypabuild.install_reqs({}, env, set(pypabuild.BASE_IGNORED_REQUIREMENTS)) # type: ignore[arg-type] + for req in pypabuild.BASE_IGNORED_REQUIREMENTS: assert req not in env.installed From 35e64ca09e5aa3da506695047d4124c9c8ed65e3 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 10 Apr 2025 17:53:07 +0530 Subject: [PATCH 05/12] Lazily load ConfigManager --- pyodide_build/build_env.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pyodide_build/build_env.py b/pyodide_build/build_env.py index b8c94029..4a13080b 100644 --- a/pyodide_build/build_env.py +++ b/pyodide_build/build_env.py @@ -15,7 +15,6 @@ from pyodide_build import __version__ from pyodide_build.common import default_xbuildenv_path, search_pyproject_toml, to_bool -from pyodide_build.config import ConfigManager, CrossBuildEnvConfigManager RUST_BUILD_PRELUDE = """ rustup default ${RUST_TOOLCHAIN} @@ -123,11 +122,20 @@ def in_xbuildenv() -> bool: return pyodide_root.name == "pyodide-root" +def _load_config_manager(): + """Lazily load ConfigManager so that we can avoid circular imports.""" + from pyodide_build.config import ConfigManager, CrossBuildEnvConfigManager + + return ConfigManager, CrossBuildEnvConfigManager + + @functools.cache def get_build_environment_vars(pyodide_root: Path) -> dict[str, str]: """ Get common environment variables for the in-tree and out-of-tree build. """ + _, CrossBuildEnvConfigManager = _load_config_manager() + config_manager = CrossBuildEnvConfigManager(pyodide_root) env = config_manager.to_env() @@ -147,6 +155,7 @@ def get_build_environment_vars(pyodide_root: Path) -> dict[str, str]: @functools.cache def get_host_build_environment_vars() -> dict[str, str]: + ConfigManager, _ = _load_config_manager() manager = ConfigManager() return manager.to_env() From 87f1276c690eb5e30400d020247d42f8c13e7a70 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 10 Apr 2025 17:59:17 +0530 Subject: [PATCH 06/12] Add tests and run [integration] --- pyodide_build/tests/test_config.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyodide_build/tests/test_config.py b/pyodide_build/tests/test_config.py index 0dd9010b..cb63fcaa 100644 --- a/pyodide_build/tests/test_config.py +++ b/pyodide_build/tests/test_config.py @@ -42,6 +42,7 @@ def test_load_config_from_file(self, tmp_path, reset_env_vars, reset_cache): default_cross_build_env_url = "https://example.com/cross_build_env.tar.gz" skip_emscripten_version_check = "1" xbuildenv_path = "my_custom/xbuildenv_path" + ignored_build_requirements = "cmake foo bar" """) config_manager = ConfigManager() @@ -54,6 +55,7 @@ def test_load_config_from_file(self, tmp_path, reset_env_vars, reset_cache): ) assert config["skip_emscripten_version_check"] == "1" assert config["xbuildenv_path"] == "my_custom/xbuildenv_path" + assert config["ignored_build_requirements"] == "cmake foo bar" class TestCrossBuildEnvConfigManager_OutOfTree: @@ -126,12 +128,14 @@ def test_load_config_from_env(self, dummy_xbuildenv, reset_env_vars, reset_cache "CMAKE_TOOLCHAIN_FILE": "/path/to/toolchain", "MESON_CROSS_FILE": "/path/to/crossfile", "PYODIDE_XBUILDENV_PATH": "/path/to/xbuildenv", + "IGNORED_BUILD_REQUIREMENTS": "cmake foo bar", } config = config_manager._load_config_from_env(env) assert config["cmake_toolchain_file"] == "/path/to/toolchain" assert config["meson_cross_file"] == "/path/to/crossfile" assert config["xbuildenv_path"] == "/path/to/xbuildenv" + assert config["ignored_build_requirements"] == "cmake foo bar" def test_load_config_from_file( self, tmp_path, dummy_xbuildenv, reset_env_vars, reset_cache @@ -151,6 +155,7 @@ def test_load_config_from_file( meson_cross_file = "$(MESON_CROSS_FILE)" build_dependency_index_url = "https://example.com/simple" xbuildenv_path = "../my_custom/xbuildenv_path" # also helps check relative paths + ignored_build_requirements = "cmake foo bar" """) xbuildenv_manager = CrossBuildEnvManager( @@ -169,6 +174,7 @@ def test_load_config_from_file( assert config["meson_cross_file"] == "/path/to/crossfile" assert config["build_dependency_index_url"] == "https://example.com/simple" assert config["xbuildenv_path"] == "../my_custom/xbuildenv_path" + assert config["ignored_build_requirements"] == "cmake foo bar" def test_config_all(self, dummy_xbuildenv, reset_env_vars, reset_cache): xbuildenv_manager = CrossBuildEnvManager( From 61362912cc6e3a25c92b4d53c7cf059fedc4ff9e Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 10 Apr 2025 18:38:38 +0530 Subject: [PATCH 07/12] Fix missing constant and env var [integration] --- pyodide_build/config.py | 1 + pyodide_build/tests/test_pypabuild.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pyodide_build/config.py b/pyodide_build/config.py index ea08c8c8..dcdafc83 100644 --- a/pyodide_build/config.py +++ b/pyodide_build/config.py @@ -238,6 +238,7 @@ def _get_make_environment_vars(self) -> Mapping[str, str]: "rust_toolchain": "nightly-2025-02-01", "rust_emscripten_target_url": "", # Other configuration + "pyodide_jobs": "1", "skip_emscripten_version_check": "0", "build_dependency_index_url": "https://pypi.anaconda.org/pyodide/simple", "default_cross_build_env_url": "", diff --git a/pyodide_build/tests/test_pypabuild.py b/pyodide_build/tests/test_pypabuild.py index 82f2a1eb..51a8356c 100644 --- a/pyodide_build/tests/test_pypabuild.py +++ b/pyodide_build/tests/test_pypabuild.py @@ -1,4 +1,5 @@ from pyodide_build import pypabuild, pywasmcross +from pyodide_build.constants import BASE_IGNORED_REQUIREMENTS class MockIsolatedEnv: @@ -27,8 +28,8 @@ def test_install_reqs(tmp_path, dummy_xbuildenv): for req in reqs: assert req in env.installed - pypabuild.install_reqs({}, env, set(pypabuild.BASE_IGNORED_REQUIREMENTS)) # type: ignore[arg-type] - for req in pypabuild.BASE_IGNORED_REQUIREMENTS: + pypabuild.install_reqs({}, env, set(BASE_IGNORED_REQUIREMENTS)) # type: ignore[arg-type] + for req in BASE_IGNORED_REQUIREMENTS: assert req not in env.installed From 1a7f3c30bae39e8ada34abda9e884da5454d7a9e Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 10 Apr 2025 19:15:09 +0530 Subject: [PATCH 08/12] Fix bad update to CHANGELOG --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbeb7ed6..ae195760 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added basic support for uv. `uv tool install pyodide-cli --with pyodide-build`, or `uvx --from pyodide-cli --with pyodide-build pyodide --help`, or using `pyodide-build` in `uv`-managed virtual environments will now work. +- Added basic support for uv. `uv tool install pyodide-cli --with pyodide-build`, or + `uvx --from pyodide-cli --with pyodide-build pyodide --help`, or using `pyodide-build` + in `uv`-managed virtual environments will now work. [#132](https://github.com/pyodide/pyodide-build/pull/132) - `pyodide build` now takes an additional `--xbuildenv-path` argument and corresponding From cfe0cf95954c101c01ef495aefa1221ee58829fa Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 12 Apr 2025 11:47:11 +0530 Subject: [PATCH 09/12] Strip whitespace in each list element Co-authored-by: Gyeongjae Choi --- pyodide_build/pypabuild.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyodide_build/pypabuild.py b/pyodide_build/pypabuild.py index c8823c18..3f4bb8d8 100644 --- a/pyodide_build/pypabuild.py +++ b/pyodide_build/pypabuild.py @@ -128,9 +128,9 @@ def remove_avoided_requirements( def install_reqs( build_env: Mapping[str, str], env: DefaultIsolatedEnv, reqs: set[str] ) -> None: - IGNORED_BUILD_REQUIREMENTS = get_host_build_flag( + IGNORED_BUILD_REQUIREMENTS = [pkg.strip() for pkg in get_host_build_flag( "IGNORED_BUILD_REQUIREMENTS" - ).split() + ).split()] # propagate PIP config from build_env to current environment with common.replace_env( os.environ | {k: v for k, v in build_env.items() if k.startswith("PIP")} From d3e5c34078d8f98a6a3e8b5718ce7f04854c60e4 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 12 Apr 2025 11:53:10 +0530 Subject: [PATCH 10/12] Add `ignored_build_requirements` back to the CLI --- pyodide_build/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyodide_build/config.py b/pyodide_build/config.py index dcdafc83..56491e24 100644 --- a/pyodide_build/config.py +++ b/pyodide_build/config.py @@ -284,4 +284,5 @@ def _get_make_environment_vars(self) -> Mapping[str, str]: "pyodide_abi_version": "PYODIDE_ABI_VERSION", "pyodide_root": "PYODIDE_ROOT", "python_include_dir": "PYTHONINCLUDE", + "ignored_build_requirements": "IGNORED_BUILD_REQUIREMENTS", } From b43d1df95a72f2a174ce1a2ab711703526e2eb30 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 12 Apr 2025 06:23:24 +0000 Subject: [PATCH 11/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyodide_build/pypabuild.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyodide_build/pypabuild.py b/pyodide_build/pypabuild.py index 3f4bb8d8..16c6442d 100644 --- a/pyodide_build/pypabuild.py +++ b/pyodide_build/pypabuild.py @@ -128,9 +128,9 @@ def remove_avoided_requirements( def install_reqs( build_env: Mapping[str, str], env: DefaultIsolatedEnv, reqs: set[str] ) -> None: - IGNORED_BUILD_REQUIREMENTS = [pkg.strip() for pkg in get_host_build_flag( - "IGNORED_BUILD_REQUIREMENTS" - ).split()] + IGNORED_BUILD_REQUIREMENTS = [ + pkg.strip() for pkg in get_host_build_flag("IGNORED_BUILD_REQUIREMENTS").split() + ] # propagate PIP config from build_env to current environment with common.replace_env( os.environ | {k: v for k, v in build_env.items() if k.startswith("PIP")} From 71f9c5e042c97a8cc6915c7f43af2d3ed2aca9bc Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 12 Apr 2025 12:03:31 +0530 Subject: [PATCH 12/12] Trigger [integration] tests