diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 4879fad67a..c47c43b06c 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -40,18 +40,21 @@ jobs: cmake_opts: '-DBUILD_SHARED_LIBS=ON -DENABLE_BENCHMARKS:BOOL=ON' do_build: 'yes' do_benchmarks: 'yes' + do_wheel: 'yes' - job_name: gcc@13.3.1, shared, 32bit host_config: gcc@13.3.1.cmake compiler_image: ${{ needs.set_image_vars.outputs.gcc_docker_image }} cmake_opts: '-DBUILD_SHARED_LIBS=ON -DAXOM_USE_64BIT_INDEXTYPE:BOOL=OFF -DAXOM_NO_INT64_T=1' do_build: 'yes' do_benchmarks: 'no' + do_wheel: 'no' - job_name: llvm@19.0.0, shared, benchmarks, quest regression host_config: llvm@19.0.0.cmake compiler_image: ${{ needs.set_image_vars.outputs.clang_docker_image }} cmake_opts: '-DBUILD_SHARED_LIBS=ON -DENABLE_BENCHMARKS:BOOL=ON' do_build: 'yes' do_benchmarks: 'yes' + do_wheel: 'no' include: - build_type: Debug config: @@ -61,6 +64,7 @@ jobs: cmake_opts: '-DBUILD_SHARED_LIBS=ON -DAXOM_ENABLE_MIR:BOOL=OFF -DAXOM_ENABLE_BUMP:BOOL=OFF -U RAJA_DIR' do_build: 'yes' do_benchmarks: 'no' + do_wheel: 'no' - build_type: Debug config: job_name: llvm@19.0.0, shared, no umpire @@ -69,6 +73,7 @@ jobs: cmake_opts: '-DBUILD_SHARED_LIBS=ON -U UMPIRE_DIR' do_build: 'yes' do_benchmarks: 'no' + do_wheel: 'no' - build_type: Debug config: job_name: llvm@19.0.0, shared, no raja and umpire @@ -77,6 +82,7 @@ jobs: cmake_opts: '-DBUILD_SHARED_LIBS=ON -DAXOM_ENABLE_MIR:BOOL=OFF -DAXOM_ENABLE_BUMP:BOOL=OFF -U RAJA_DIR -U UMPIRE_DIR' do_build: 'yes' do_benchmarks: 'no' + do_wheel: 'no' - build_type: Debug config: job_name: llvm@19.0.0, shared, no profiling @@ -85,6 +91,7 @@ jobs: cmake_opts: '-DBUILD_SHARED_LIBS=ON -U CALIPER_DIR -U ADIAK_DIR' do_build: 'yes' do_benchmarks: 'no' + do_wheel: 'no' name: ${{ matrix.build_type }} - ${{ matrix.config.job_name }} container: image: ${{ matrix.config.compiler_image }} @@ -113,6 +120,18 @@ jobs: CMAKE_EXTRA_FLAGS=" ${{ matrix.config.cmake_opts }} -DAXOM_QUEST_ENABLE_EXTRA_REGRESSION_TESTS:BOOL=${{ contains(matrix.config.job_name, 'quest regression') && 'ON' || 'OFF' }} " \ BUILD_TYPE=${{ matrix.build_type }} \ ./scripts/github-actions/linux-build_and_test.sh + - name: Build and test the Python wheel - ${{ matrix.build_type }} - ${{ matrix.config.job_name }} + # Builds Python wheel (src/python) against the Axom that the step above just installed; one build type is enough. + if: matrix.config.do_wheel == 'yes' && matrix.build_type == 'Release' + shell: bash + timeout-minutes: 40 + run: | + # Find the installation path + AXOM_INSTALL=$(awk -F= '/^CMAKE_INSTALL_PREFIX:PATH=/{print $2}' builddir/CMakeCache.txt) + # Run the wheels test + HOST_CONFIG=host-configs/docker/${{ matrix.config.host_config }} \ + AXOM_INSTALL="${AXOM_INSTALL}" \ + ./scripts/github-actions/linux-wheel_and_test.sh - name: Upload Test Results uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 with: diff --git a/.gitignore b/.gitignore index e918142159..a0b27222da 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ install- .project .settings build-* +build/ install-* _axom_build_and_test_* *.pyc diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index e961902bf2..af782d85ad 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -61,6 +61,8 @@ The Axom project release numbers follow [Semantic Versioning](http://semver.org/ - Klee: Adds support for lua-based input decks for shaping - Slam: Adds convenience aliases in `axom/slam/Aliases.hpp` for the most common set and relation configurations, including `ArraySet`, `ArrayViewSet`, `VariableRelation`, `ConstantRelation` and their `View` forms. +- Python: Adds a scikit-build-core project under `src/python/` for building a thin, pip/uv-installable `axom` wheel. + The wheel compiles Axom's bindings against an already-installed Axom. ### Removed - Bump: Removed `axom::bump::views::MultiBufferMaterialView`, which was a view type for an obsolete flavor of Blueprint matset. @@ -104,6 +106,8 @@ The Axom project release numbers follow [Semantic Versioning](http://semver.org/ - Quest: Status-returning reader/writer operations in `C2CReader`, `MFEMReader`, `ProEReader`, `STEPReader`, `STLReader`, `STLWriter`, and their parallel variants are now marked `[[nodiscard]]`. Callers that previously ignored returned status values must check them to avoid compiler diagnostics. +- Python: Sidre's bindings all name their arguments, so they can be passed by keyword and show up in IDE + completion and signature help. ### Fixed - MIR/Bump: `MergeCoordsetPoints` now only emits its node-merge `SLIC_INFO` when MIR `verbose` is enabled on the Conduit options passed through ELVIRA. diff --git a/scripts/github-actions/linux-wheel_and_test.sh b/scripts/github-actions/linux-wheel_and_test.sh new file mode 100755 index 0000000000..6bc8aa1e7a --- /dev/null +++ b/scripts/github-actions/linux-wheel_and_test.sh @@ -0,0 +1,166 @@ +#!/bin/bash +############################################################################## +# Copyright (c) Lawrence Livermore National Security, LLC and other +# Axom Project Contributors. See top-level LICENSE and COPYRIGHT +# files for dates and other details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################## + +# Build the thin, pip/uv-installable Axom wheel and exercise it end to end, +# without using the run_python_with_axom.sh wrapper or updating the PYTHONPATH: +# +# 1. build the wheel from src/python against the prebuilt Axom install +# specified by AXOM_DIR or AXOM_INSTALL (find_package(axom)); +# 2. install the wheel into a fresh uv venv; +# 3. verify the wheel installed conduit.pth for the same-build Conduit python module; +# 4. run the Sidre Python test suite with plain pytest. +# +# Intended for the gcc docker image, which is nanobind-enabled. + +# Fail on the first error, including inside pipelines, and trace every command so +# a CI failure is readable from the log alone. +set -e +set -o pipefail +set -x + +HOST_CONFIG="${HOST_CONFIG:-host-configs/docker/gcc@13.3.1.cmake}" + +echo "~~~~ helpful info ~~~~" +echo "USER=$(id -u -n)" +echo "PWD=$(pwd)" +echo "HOST_CONFIG=${HOST_CONFIG}" +echo "~~~~~~~~~~~~~~~~~~~~~~" + +absolute_path() { + local path="$1" + if [[ ! -e "${path}" ]]; then + echo "ERROR: Path does not exist: ${path}" >&2 + return 1 + fi + local dir + local base + dir=$(dirname "${path}") + base=$(basename "${path}") + printf "%s/%s" "$(cd "${dir}" && pwd -P)" "${base}" +} + +if [[ ! -f "${HOST_CONFIG}" ]]; then + echo "ERROR: Host-config not found: ${HOST_CONFIG}" >&2 + exit 1 +fi +HOST_CONFIG_PATH=$(absolute_path "${HOST_CONFIG}") +echo "HOST_CONFIG_PATH=${HOST_CONFIG_PATH}" + +# extract value from host-config line of the form `set(${name} ON CACHE BOOL "")` +# then capitalizes it and looks for true-like patterns +cmake_bool_from_file_is_on() { + local name="$1" + local value + value=$(awk -v name="${name}" ' + $0 ~ "set\\(" name "[ \t\"]+" { + line = $0 + sub("^[ \t]*set\\(" name "[ \t\"]+", "", line) + sub("[ \t\"\\)].*$", "", line) + print line + exit + } + ' "${HOST_CONFIG_PATH}") + value="${value^^}" + [[ "${value}" == "ON" || "${value}" == "TRUE" || "${value}" == "YES" || "${value}" == "1" ]] +} + +if [[ -n "${AXOM_INSTALL:-}" && -z "${AXOM_DIR:-}" ]]; then + AXOM_DIR="${AXOM_INSTALL%/}/lib/cmake" +fi + +if [[ -z "${AXOM_DIR:-}" || ! -f "${AXOM_DIR}/axom-config.cmake" ]]; then + echo "ERROR: Axom CMake package not found." >&2 + echo " Set AXOM_DIR to the directory containing axom-config.cmake," >&2 + echo " or set AXOM_INSTALL to an Axom install prefix." >&2 + exit 1 +fi +AXOM_DIR=$(absolute_path "${AXOM_DIR}") +echo "AXOM_DIR=${AXOM_DIR}" + +AXOM_WHEEL_ENABLE_MPI=OFF +if cmake_bool_from_file_is_on ENABLE_MPI; then + AXOM_WHEEL_ENABLE_MPI=ON +fi +echo "AXOM_WHEEL_ENABLE_MPI=${AXOM_WHEEL_ENABLE_MPI}" + +echo "~~~~~~ ENSURE uv IS AVAILABLE ~~~~~~" +if ! command -v uv >/dev/null 2>&1; then + # Distro Pythons (e.g. Ubuntu 24.04) ship a PEP 668 EXTERNALLY-MANAGED marker, which makes `pip install --user` fail. + # uv only lands in the user site directory so opting out is safe here. + pip_args="--user" + if python3 -c 'import os, sysconfig, sys; sys.exit(0 if os.path.exists(os.path.join(sysconfig.get_path("stdlib"), "EXTERNALLY-MANAGED")) else 1)'; then + pip_args="${pip_args} --break-system-packages" + fi + python3 -m pip install ${pip_args} uv + export PATH="${HOME}/.local/bin:${PATH}" +fi +uv --version + +echo "~~~~~~ BUILD THE THIN WHEEL FROM src/python ~~~~~~" +# Point find_package at the install with AXOM_DIR +# Conduit resolves transitively from axom's config, which records its Conduit prefix +rm -rf dist +uv build --wheel \ + -C cmake.args=-C \ + -C "cmake.args=${HOST_CONFIG_PATH}" \ + -C "cmake.define.AXOM_DIR=${AXOM_DIR}" \ + --out-dir dist \ + src/python +ls -l dist +AXOM_WHEEL=$(find dist -maxdepth 1 -name 'axom-*.whl' -print -quit) +if [[ -z "${AXOM_WHEEL}" ]]; then + echo "ERROR: Axom wheel not found in dist/." + exit 1 +fi + +echo "~~~~~~ FRESH VENV + INSTALL THE WHEEL ~~~~~~" +# Pin the interpreter that built the wheel, so the venv cannot pick a different one. +VENV_DIR=/tmp/axom-wheel-venv +rm -rf "${VENV_DIR}" +uv venv --python "$(command -v python3)" "${VENV_DIR}" +VENV_PY="${VENV_DIR}/bin/python" +AXOM_WHEEL_EXTRAS="test" +if [[ "${AXOM_WHEEL_ENABLE_MPI}" == "ON" ]]; then + AXOM_WHEEL_EXTRAS="test,mpi" +fi +uv pip install --python "${VENV_PY}" "${AXOM_WHEEL}[${AXOM_WHEEL_EXTRAS}]" + +echo "~~~~~~ VERIFY WHEEL-INSTALLED CONDUIT .pth ~~~~~~" +PLATLIB=$("${VENV_PY}" -c 'import sysconfig; print(sysconfig.get_paths()["platlib"])') +CONDUIT_PTH="${PLATLIB}/conduit.pth" +if [[ ! -f "${CONDUIT_PTH}" ]]; then + echo "ERROR: Expected wheel to install ${CONDUIT_PTH}." + echo " The wheel should expose the same-build Conduit python module without a manual PYTHONPATH update." + exit 1 +fi +CONDUIT_PY_DIR=$(sed -n '1p' "${CONDUIT_PTH}") +if [[ -z "${CONDUIT_PY_DIR}" || ! -d "${CONDUIT_PY_DIR}" ]]; then + echo "ERROR: ${CONDUIT_PTH} points to missing Conduit python module directory '${CONDUIT_PY_DIR}'." + exit 1 +fi +echo "verified ${CONDUIT_PTH} -> ${CONDUIT_PY_DIR}" + +echo "~~~~~~ IMPORT SMOKE TEST ~~~~~~" +"${VENV_PY}" -c \ + "import axom, axom.sidre, conduit, numpy; print('axom', axom.__version__); print('axom.sidre', axom.sidre.__version__)" +if [[ "${AXOM_WHEEL_ENABLE_MPI}" == "ON" ]]; then + "${VENV_PY}" -c "import mpi4py, axom.sidre as sidre; assert sidre.AXOM_ENABLE_MPI" +else + "${VENV_PY}" -c "import axom.sidre as sidre; assert not sidre.AXOM_ENABLE_MPI" +fi + +echo "~~~~~~ RUN THE SIDRE PYTHON SUITE VIA PLAIN pytest ~~~~~~" +# Axom's Python tests are named *_Py.py, which pytest's default python_files patterns do not match +TEST_DIR="$(pwd)/src/axom/sidre/tests" +SCRATCH="$(mktemp -d)" +pushd "${SCRATCH}" > /dev/null +"${VENV_PY}" -m pytest -s -p no:cacheprovider \ + -o python_files='*_Py.py' \ + "${TEST_DIR}" +popd > /dev/null diff --git a/src/axom/sidre/CMakeLists.txt b/src/axom/sidre/CMakeLists.txt index 2091cb2fdd..360d78ddb0 100644 --- a/src/axom/sidre/CMakeLists.txt +++ b/src/axom/sidre/CMakeLists.txt @@ -208,6 +208,8 @@ if(NANOBIND_FOUND) "${_axom_py_build_root}/axom/__init__.py" COPYONLY) axom_configure_file("${_axom_py_pkg_src}/axom/py.typed" "${_axom_py_build_root}/axom/py.typed" COPYONLY) + axom_configure_file("${_axom_py_pkg_src}/axom/config.py" + "${_axom_py_build_root}/axom/config.py" COPYONLY) axom_configure_file("${_axom_py_pkg_src}/axom/sidre/__init__.py" "${_axom_py_build_root}/axom/sidre/__init__.py" COPYONLY) # Hand-written package stub: re-exports the generated _sidre.pyi statically @@ -249,6 +251,7 @@ if(NANOBIND_FOUND) # Namespace-root package files install once (not per component). install(FILES "${_axom_py_pkg_src}/axom/__init__.py" "${_axom_py_pkg_src}/axom/py.typed" + "${_axom_py_pkg_src}/axom/config.py" DESTINATION "${AXOM_PYTHON_MODULE_INSTALL_PREFIX}/axom") endif() diff --git a/src/axom/sidre/docs/sphinx/python_interface.rst b/src/axom/sidre/docs/sphinx/python_interface.rst index f0c91e283a..08d4a6b23f 100644 --- a/src/axom/sidre/docs/sphinx/python_interface.rst +++ b/src/axom/sidre/docs/sphinx/python_interface.rst @@ -89,19 +89,133 @@ After ``spack install``, the environment's interpreter should have a working Axo pip / uv wheel (thin, external Axom) ------------------------------------ +The wheel compiles only the Sidre binding against an already-installed Axom. +It is tied to that Axom install, its Conduit install, and its host-config; +it is not a portable PyPI-style wheel. + .. note:: + **Do not install Conduit from PyPI.** ``axom.sidre`` must use the same + ``libconduit`` that Axom was built against. The PyPI packages named + ``conduit`` and ``llnl-conduit`` do not provide that same build. + + Use the Conduit Python package from the Conduit install recorded by Axom. + Wheels built by Axom's Python project record that path in ``conduit.pth``. + +Quick start +^^^^^^^^^^^ + +Use an absolute ``AXOM_DIR`` pointing at the directory containing +``axom-config.cmake``, usually ``$AXOM_INSTALL/lib/cmake``. + +.. code-block:: bash + + $ uv venv --python $(which python3) + + $ uv pip install /path/to/axom/src/python \ + -C cmake.define.AXOM_DIR="$AXOM_INSTALL/lib/cmake" + + $ uv run python -c "import axom.sidre, conduit, numpy; print(axom.__version__)" + +Optional dependencies use the normal Python extras syntax on the local source +path. Keep the same CMake ``-C`` options used for the Axom install: + +.. code-block:: bash + + $ uv pip install '/path/to/axom/src/python[mpi]' \ + -C cmake.define.AXOM_DIR="$AXOM_INSTALL/lib/cmake" + + $ uv pip install '/path/to/axom/src/python[test]' \ + -C cmake.define.AXOM_DIR="$AXOM_INSTALL/lib/cmake" + +Use ``[mpi]`` for ``mpi4py`` support, ``[test]`` for ``pytest``, +or combine extras as ``'/path/to/axom/src/python[mpi,test]'``. +If the Axom wheel is already installed and you only need the optional dependency package, +installing ``mpi4py`` or ``pytest`` directly is also fine. + +If ``axom.sidre`` is already installed in a venv but ``import conduit`` fails, +add the same-build Conduit Python package with one ``.pth`` file. +An Axom install records this path as ``AXOM_CONDUIT_PYTHON_MODULE_DIR`` in ``axom-config.cmake``: + +.. code-block:: bash + + $ CONDUIT_PY_DIR=/path/to/conduit/install/python-modules + $ printf '%s\n' "$CONDUIT_PY_DIR" > \ + "$(uv run python -c 'import sysconfig; print(sysconfig.get_paths()["platlib"])')/axom-conduit.pth" + $ uv run python -c "import axom.sidre, conduit; print(conduit.__file__)" + +If your site publishes a host-config-specific wheelhouse, install from the path +they provide with ``uv pip install axom --find-links ``. +Axom does not assume a central wheelhouse. + +The installed wheel also carries a CMake host-config for downstream projects: + +.. code-block:: bash + + $ cmake -C "$(uv run axom-python-config --host-config)" -S /path/to/project -B build + +For build details, including MPI compiler wrappers, editable installs, +and stable ABI wheels, see ``src/python/README.md``. + +Using Axom in Jupyter +^^^^^^^^^^^^^^^^^^^^^ + +Because the wheel and the Conduit ``.pth`` live in the venv's ``site-packages``, +a Jupyter kernel running in that venv imports ``axom.sidre`` natively -- there is +nothing extra to configure, and no need to modify ``PYTHONPATH``. +Add Jupyter to the same venv and register it as a kernel: + +.. code-block:: bash + + $ uv pip install jupyterlab ipykernel + $ uv run python -m ipykernel install --user --name axom --display-name "Axom (uv)" + $ uv run jupyter lab + +For more IDE-like completions, signature help, and hover documentation in JupyterLab, +install the language-server packages in the same venv: + +.. code-block:: bash + + $ uv pip install jupyterlab-lsp 'python-lsp-server[all]' + +The Axom wheel installs PEP 561 type information and generated ``.pyi`` stubs for ``axom.sidre``. +JupyterLab's LSP extension can use those stubs for richer completion and overload help +than the classic notebook frontend usually shows. + +Select the **Axom (uv)** kernel, then for example: + +.. code-block:: python + + import axom.sidre as sidre + import numpy as np + + ds = sidre.DataStore() + grp = ds.getRoot().createGroup("fields") + view = grp.createViewAndAllocate("velocity", sidre.TypeID.FLOAT64_ID, 4) + np.asarray(view.getDataArray())[:] = [1.0, 2.0, 3.0, 4.0] # zero-copy view + print(np.asarray(grp.getView("velocity").getDataArray())) + +.. warning:: + + Sidre currently preserves the C++ API's no-op semantics for some invalid operations. + For example, ``grp.createGroup("foo")`` followed by another ``grp.createGroup("foo")`` + returns ``None`` for the second call unless ``accept_existing=True`` is passed. + The related SLIC diagnostic may be written to the process stderr/log stream + instead of appearing as a notebook cell error, so notebook code should either check for ``None`` + or use the explicit ``accept_existing`` option when reusing a group is intended. + +If the kernel cannot import ``axom.sidre``, it is nearly always either the wrong kernel +(one outside the venv) or a missing Conduit ``.pth``. Check both from inside the notebook: + +.. code-block:: python + + import sys; print(sys.executable) # expect /bin/python + import conduit; print(conduit.__file__) # expect $CONDUIT_INSTALL/lib/pythonX.Y/site-packages/... - The pip/uv-installable wheel is planned and not yet available. - This section is a placeholder for the workflow it will enable. - Until it lands, use the build-tree helper for development builds - or a dedicated Spack environment view for installed-package testing. - -The wheel will compile only the binding code against an already-installed Axom -(located via ``CMAKE_PREFIX_PATH``); it will not build Axom or its third-party -libraries. Because a pip-built Conduit would produce a second, ABI-incompatible -``libconduit`` in the same process, the wheel will rely on the Conduit Python -module from the same Axom/Conduit build, exposed via a ``.pth`` file rather -than a PyPI install. +If the underlying Axom is an MPI build and you need to pass a communicator to +``IOManager`` (or to initialize MPI), install the ``mpi`` extra. +For a local source install, use ``uv pip install '/path/to/axom/src/python[mpi]' -C ...`` +as shown above; for a prebuilt wheel from a wheelhouse, +use ``uv pip install 'axom[mpi]' --find-links ``. ==================================== Working with Conduit and NumPy diff --git a/src/axom/sidre/nanobind_sidre.cpp b/src/axom/sidre/nanobind_sidre.cpp index 464d30398e..ee30f8ec0a 100644 --- a/src/axom/sidre/nanobind_sidre.cpp +++ b/src/axom/sidre/nanobind_sidre.cpp @@ -11,6 +11,8 @@ #include #include +#include +#include #include #include #include @@ -19,13 +21,13 @@ #include "axom/core/Types.hpp" #include "axom/slic/interface/slic.hpp" -#include "core/SidreTypes.hpp" -#include "core/Buffer.hpp" -#include "core/View.hpp" -#include "core/DataStore.hpp" -#include "core/Group.hpp" +#include "axom/sidre/core/SidreTypes.hpp" +#include "axom/sidre/core/Buffer.hpp" +#include "axom/sidre/core/View.hpp" +#include "axom/sidre/core/DataStore.hpp" +#include "axom/sidre/core/Group.hpp" #if defined(AXOM_USE_MPI) - #include "spio/IOManager.hpp" + #include "axom/sidre/spio/IOManager.hpp" #endif // Separate Conduit header for python functionality @@ -351,12 +353,119 @@ DataStore* owningDataStore(View* view) //! Erase all pins recorded for \a ds (called when the DataStore is collected). void releaseDataStoreExternalPins(DataStore* ds) { externalDataOwnerRegistry().erase(ds); } +//! Release the pin recorded for \a view, if any (defined below). +void releaseExternalDataOwner(View* view); + +/*! + * \brief True when \a ptr already points into storage owned by a Buffer of \a ds. + * + * Such storage cannot dangle: Sidre owns it, and it outlives any Python proxy. + * Pinning it would cause problems, as described on pinExternalDataOwner() below. + * + * \note The scan is linear in the number of Buffers in \a ds and runs once per external-data pin + * (i.e. per createView/setExternalData call that supplies an ndarray), so creating many external views + * in a DataStore that also holds many Buffers could be expensive. + * + * \note Scoped to Buffers of \a ds only. A pointer into another DataStore's Buffer + * is not tracked by this registry, and would still need a pin. + */ +bool isOwnedByDataStoreBuffer(DataStore* ds, const void* ptr) +{ + if(ds == nullptr || ptr == nullptr) + { + return false; + } + + const auto p = reinterpret_cast(ptr); + for(auto& buffer : ds->buffers()) + { + const void* base_ptr = buffer.getVoidPtr(); + if(base_ptr == nullptr) + { + continue; + } + const auto base = reinterpret_cast(base_ptr); + const auto bytes = static_cast(buffer.getTotalBytes()); + if(p >= base && p < base + bytes) + { + return true; + } + } + return false; +} + +/*! + * \brief Return the pin already recorded in \a ds whose storage contains \a ptr, else nullptr. + * + * Used to redirect a pin away from an array that is merely a window onto storage + * this DataStore already pins. See the discussion on pinExternalDataOwner(). + * + * \note The scan is linear in the number of pins recorded for \a ds, alongside the + * Buffer scan in isOwnedByDataStoreBuffer(), and runs once per external-data pin. + * + * \note The returned pointer is into the registry's map and is invalidated by the + * next insertion, so callers must copy the ndarray before modifying the map. + */ +const nb::ndarray<>* findExistingPinOwning(DataStore* ds, const void* ptr) +{ + if(ds == nullptr || ptr == nullptr) + { + return nullptr; + } + + auto entry = externalDataOwnerRegistry().find(ds); + if(entry == externalDataOwnerRegistry().end()) + { + return nullptr; + } + + const auto p = reinterpret_cast(ptr); + for(const auto& pin : entry->second.pins) + { + const void* base_ptr = pin.second.data(); + if(base_ptr == nullptr) + { + continue; + } + const auto base = reinterpret_cast(base_ptr); + const auto bytes = static_cast(pin.second.nbytes()); + if(p >= base && p < base + bytes) + { + return &pin.second; + } + } + return nullptr; +} + /*! * \brief Record \a owner as the pin for \a view, scoped to its DataStore. * * On the first pin into a given DataStore, installs a weak reference on the * DataStore's Python object so the sub-map is cleared when the DataStore is * destroyed. Re-assigning a View*'s pin releases the previous ndarray wrapper. + * + * \note Storage that Sidre already owns is deliberately *not* pinned. + * Pinning it would create a reference cycle that this registry cannot break: + * the pin holds a strong reference to the ndarray, an ndarray produced by Buffer/View.getDataArray() + * transitively holds a strong reference to that Sidre object's Python wrapper, + * and that wrapper keeps the DataStore's Python object alive. But this is the + * object whose collection is supposed to fire the weakref callback that releases the pin. + * The cycle runs through this C++ registry, so Python's cyclic collector cannot see or break it, + * and the DataStore, Group, View and Buffer would be retained for the life of the process + * (nanobind reports them at shutdown as leaked instances). + * The idiom that triggers it is common: `data = view.getBuffer().getDataArray()` + * followed by `group.createView("name", data)`. Skipping the pin is safe because the + * Buffer owns that storage; the dangling-pointer hazard the pin exists to prevent + * only arises for storage owned by a Python object. + * + * \note The same cycle also arises one step removed, when the array is a window onto + * storage this DataStore already pins -- `arr = external_view.getDataArray()` followed + * by `group.createView("name", arr)`. Here the storage is *not* Sidre-owned, so a pin is + * genuinely needed, but `arr` is owned by the source View's Python wrapper and pinning it + * would retain the DataStore just as above. The pin is therefore redirected to the + * original owner recorded for that storage (see findExistingPinOwning()), which owns the + * memory and holds no Sidre reference. The redirect is per-DataStore, so aliasing another + * DataStore's external storage still pins the array as given. */ void pinExternalDataOwner(View* view, const nb::ndarray<>& owner) { @@ -370,6 +479,26 @@ void pinExternalDataOwner(View* view, const nb::ndarray<>& owner) return; } + // Sidre-owned storage needs no pin, and pinning it would leak the DataStore + if(isOwnedByDataStoreBuffer(ds, owner.data())) + { + // Drop any pin a previous, non-Sidre-owned array left on this View. + releaseExternalDataOwner(view); + return; + } + + // The array may be a window onto storage this DataStore already pins, e.g. + // `arr = external_view.getDataArray()` followed by `group.createView(name, arr)`. + // Such an array is owned by the source View's Python wrapper, so pinning it + // recreates the cycle described above. Pin the original owner instead: it is + // the object that actually owns the memory and it holds no Sidre reference. + // Copy it out before touching the map, which may rehash. + nb::ndarray<> pinned(owner); + if(const nb::ndarray<>* existing = findExistingPinOwning(ds, owner.data())) + { + pinned = nb::ndarray<>(*existing); + } + DataStoreExternalPins& entry = externalDataOwnerRegistry()[ds]; if(!entry.datastore_weakref.is_valid()) { @@ -387,7 +516,7 @@ void pinExternalDataOwner(View* view, const nb::ndarray<>& owner) } // Map assignment releases the previous ndarray wrapper if one was present. - entry.pins[view] = nb::ndarray<>(owner); + entry.pins[view] = pinned; } void releaseExternalDataOwner(View* view) @@ -659,8 +788,14 @@ NB_MODULE(_sidre, m_sidre) m_sidre.attr("InvalidIndex") = axom::InvalidIndex; m_sidre.attr("InvalidName") = axom::utilities::string::InvalidName; - m_sidre.def("indexIsValid", &indexIsValid, "Returns true if idx is valid, else false."); - m_sidre.def("nameIsValid", &nameIsValid, "Returns true if name is valid, else false."); + m_sidre.def("indexIsValid", + &indexIsValid, + "Returns true if idx is valid, else false.", + nb::arg("idx")); + m_sidre.def("nameIsValid", + &nameIsValid, + "Returns true if name is valid, else false.", + nb::arg("name")); #if defined(AXOM_USE_HDF5) m_sidre.attr("AXOM_USE_HDF5") = true; @@ -727,11 +862,13 @@ NB_MODULE(_sidre, m_sidre) .def("getNumBuffers", &DataStore::getNumBuffers, "Return number of Buffers in the DataStore") .def("hasBuffer", &DataStore::hasBuffer, - "Return true if DataStore owns a Buffer with given index; else false") + "Return true if DataStore owns a Buffer with given index; else false", + nb::arg("idx")) .def("getBuffer", &DataStore::getBuffer, nb::rv_policy::reference_internal, - "Return pointer to Buffer object with the given index") + "Return pointer to Buffer object with the given index", + nb::arg("idx")) .def("createBuffer", nb::overload_cast<>(&DataStore::createBuffer), @@ -740,13 +877,17 @@ NB_MODULE(_sidre, m_sidre) .def("createBuffer", nb::overload_cast(&DataStore::createBuffer), nb::rv_policy::reference_internal, - "Create a Buffer object with specified type and number of elements") + "Create a Buffer object with specified type and number of elements", + nb::arg("type"), + nb::arg("num_elems")) .def("destroyBuffer", nb::overload_cast(&DataStore::destroyBuffer), - "Remove Buffer from the DataStore and destroy it and its data") + "Remove Buffer from the DataStore and destroy it and its data", + nb::arg("buffer")) .def("destroyBuffer", nb::overload_cast(&DataStore::destroyBuffer), - "Remove Buffer with given index from the DataStore and destroy it and its data.") + "Remove Buffer with given index from the DataStore and destroy it and its data.", + nb::arg("idx")) .def("destroyAllBuffers", &DataStore::destroyAllBuffers, "Remove all Buffers from the DataStore and destroy them and their data") @@ -755,12 +896,17 @@ NB_MODULE(_sidre, m_sidre) "Return first valid Buffer index") .def("getNextValidBufferIndex", &DataStore::getNextValidBufferIndex, - "Return next valid Buffer index after given index") + "Return next valid Buffer index after given index", + nb::arg("idx")) .def("generateBlueprintIndex", nb::overload_cast( &DataStore::generateBlueprintIndex), - "Generate a Conduit Blueprint index based on a mesh in stored in this DataStore.") + "Generate a Conduit Blueprint index based on a mesh in stored in this DataStore.", + nb::arg("domain_path"), + nb::arg("mesh_name"), + nb::arg("index_path"), + nb::arg("num_domains")) .def("buffers", nb::overload_cast<>(&DataStore::buffers), nb::keep_alive<0, 1>(), @@ -784,33 +930,42 @@ NB_MODULE(_sidre, m_sidre) .def("createAttributeString", &DataStore::createAttributeString, nb::rv_policy::reference_internal, - "Create an Attribute object with a default string value") + "Create an Attribute object with a default string value", + nb::arg("name"), + nb::arg("default_value").noconvert()) .def("hasAttribute", nb::overload_cast(&DataStore::hasAttribute, nb::const_), - "Return true if DataStore has created attribute name, else false") + "Return true if DataStore has created attribute name, else false", + nb::arg("name")) .def("hasAttribute", nb::overload_cast(&DataStore::hasAttribute, nb::const_), - "Return true if DataStore has created attribute with index, else false") + "Return true if DataStore has created attribute with index, else false", + nb::arg("idx")) .def("destroyAttribute", nb::overload_cast(&DataStore::destroyAttribute), - "Remove Attribute from the DataStore and destroy it and its data") + "Remove Attribute from the DataStore and destroy it and its data", + nb::arg("name")) .def("destroyAttribute", nb::overload_cast(&DataStore::destroyAttribute), - "Remove Attribute with given index from the DataStore and destroy it and its data") + "Remove Attribute with given index from the DataStore and destroy it and its data", + nb::arg("idx")) .def("destroyAttribute", nb::overload_cast(&DataStore::destroyAttribute), - "Remove Attribute from the DataStore and destroy it and its data") + "Remove Attribute from the DataStore and destroy it and its data", + nb::arg("attr")) .def("destroyAllAttributes", &DataStore::destroyAllAttributes, "Remove all Attributes from the DataStore and destroy them and their data") .def("getAttribute", nb::overload_cast(&DataStore::getAttribute), nb::rv_policy::reference_internal, - "Return pointer to non-const Attribute with given index") + "Return pointer to non-const Attribute with given index", + nb::arg("idx")) .def("getAttribute", nb::overload_cast(&DataStore::getAttribute), nb::rv_policy::reference_internal, - "Return pointer to non-const Attribute with given name") + "Return pointer to non-const Attribute with given name", + nb::arg("name")) // Requires conduit::Node information // .def("saveAttributeLayout", @@ -827,7 +982,8 @@ NB_MODULE(_sidre, m_sidre) .def("getNextValidAttributeIndex", &DataStore::getNextValidAttributeIndex, "Return next valid Attribute index in DataStore object after given index" - "(i.e., smallest index over all Attribute indices larger than given one)") + "(i.e., smallest index over all Attribute indices larger than given one)", + nb::arg("idx")) .def("attributes", nb::overload_cast<>(&DataStore::attributes), nb::keep_alive<0, 1>(), @@ -969,7 +1125,9 @@ NB_MODULE(_sidre, m_sidre) "Return number of dimensions in data view and shape information" " of this data view object." " ndims - maximum number of dimensions to return." - " shape - user supplied numpy 1D array assumed to be ndims long.") + " shape - user supplied numpy 1D array assumed to be ndims long.", + nb::arg("ndims"), + nb::arg("shape")) .def("allocate", nb::overload_cast(&View::allocate), @@ -986,7 +1144,8 @@ NB_MODULE(_sidre, m_sidre) .def("reallocate", nb::overload_cast(&View::reallocate), nb::rv_policy::reference, - "Reallocate data for the View.") + "Reallocate data for the View.", + nb::arg("num_elems")) .def("attachBuffer", nb::overload_cast(&View::attachBuffer), nb::rv_policy::reference, @@ -1037,7 +1196,10 @@ NB_MODULE(_sidre, m_sidre) return self.apply(type, ndims, shape.data()); }, nb::rv_policy::reference, - "Apply data description with type and numpy shape.") + "Apply data description with type and numpy shape.", + nb::arg("type"), + nb::arg("ndims"), + nb::arg("shape")) .def("setScalar", &View::setScalar, nb::rv_policy::reference, @@ -1080,7 +1242,10 @@ NB_MODULE(_sidre, m_sidre) return setExternalDataAndPinOwner(self, type, num_elems, external_ptr); }, nb::rv_policy::reference, - "Set the View to hold described external data (numpy array).") + "Set the View to hold described external data (numpy array).", + nb::arg("type"), + nb::arg("num_elems"), + nb::arg("external_ptr")) .def( "setExternalData", [](View& self, @@ -1091,7 +1256,11 @@ NB_MODULE(_sidre, m_sidre) return setExternalDataAndPinOwner(self, type, ndims, shape, external_ptr); }, nb::rv_policy::reference, - "Set the View to hold described external data (numpy array).") + "Set the View to hold described external data (numpy array).", + nb::arg("type"), + nb::arg("ndims"), + nb::arg("shape"), + nb::arg("external_ptr")) .def("getString", &View::getString, @@ -1119,24 +1288,28 @@ NB_MODULE(_sidre, m_sidre) .def("print", nb::overload_cast<>(&View::print, nb::const_), "Print JSON description of the View.") - .def("rename", &View::rename, "Change the name of the View.") + .def("rename", &View::rename, "Change the name of the View.", nb::arg("new_name")) // Attribute accessors .def("getAttribute", nb::overload_cast(&View::getAttribute), nb::rv_policy::reference_internal, - "Get Attribute by index") + "Get Attribute by index", + nb::arg("idx")) .def("getAttribute", nb::overload_cast(&View::getAttribute), nb::rv_policy::reference_internal, - "Get Attribute by name") + "Get Attribute by name", + nb::arg("name")) .def("hasAttributeValue", nb::overload_cast(&View::hasAttributeValue, nb::const_), - "Return true if the attribute (by index) has been explicitly set; else false.") + "Return true if the attribute (by index) has been explicitly set; else false.", + nb::arg("idx")) .def("hasAttributeValue", nb::overload_cast(&View::hasAttributeValue, nb::const_), - "Return true if the attribute (by name) has been explicitly set; else false.") + "Return true if the attribute (by name) has been explicitly set; else false.", + nb::arg("name")) .def("hasAttributeValue", nb::overload_cast(&View::hasAttributeValue, nb::const_), nb::arg("attr").none(), @@ -1144,59 +1317,86 @@ NB_MODULE(_sidre, m_sidre) .def("setAttributeToDefault", nb::overload_cast(&View::setAttributeToDefault), - "Set Attribute (by index) to its default value") + "Set Attribute (by index) to its default value", + nb::arg("idx")) .def("setAttributeToDefault", nb::overload_cast(&View::setAttributeToDefault), - "Set Attribute (by name) to its default value") + "Set Attribute (by name) to its default value", + nb::arg("name")) .def("setAttributeToDefault", nb::overload_cast(&View::setAttributeToDefault), nb::arg("attr").none(), "Set Attribute (by pointer) to its default value") - // Scalar setters for int and python float (C++ double) + // Scalar setters for int and python float (C++ double). + // + // NOTE: the value argument is bound with .noconvert(), so nanobind skips its + // converting overload pass and only an exact python int or float is accepted. + // This is deliberate. With conversion enabled, nanobind tries the overloads in declaration order, + // so a numpy float binds to the int overload and the value is silently truncated + // (e.g. np.float32(3.5) stored as 3). Rejecting the call is better than storing the wrong number. + // Callers holding a numpy scalar convert at the call site, e.g. int(x), float(x) or x.item(). .def( "setAttributeScalar", [](View& self, IndexType idx, int value) { return self.setAttributeScalar(idx, value); }, - "Set Attribute (by index) to int value") + "Set Attribute (by index) to int value", + nb::arg("idx"), + nb::arg("value").noconvert()) .def( "setAttributeScalar", [](View& self, IndexType idx, double value) { return self.setAttributeScalar(idx, value); }, - "Set Attribute (by index) to float (C++ double) value") + "Set Attribute (by index) to float (C++ double) value", + nb::arg("idx"), + nb::arg("value").noconvert()) .def( "setAttributeScalar", [](View& self, const std::string& name, int value) { return self.setAttributeScalar(name, value); }, - "Set Attribute (by name) to int value") + "Set Attribute (by name) to int value", + nb::arg("name"), + nb::arg("value").noconvert()) .def( "setAttributeScalar", [](View& self, const std::string& name, double value) { return self.setAttributeScalar(name, value); }, - "Set Attribute (by name) to float (C++ double) value") + "Set Attribute (by name) to float (C++ double) value", + nb::arg("name"), + nb::arg("value").noconvert()) .def( "setAttributeScalar", [](View& self, const Attribute* attr, int value) { return self.setAttributeScalar(attr, value); }, - "Set Attribute (by pointer) to int value") + "Set Attribute (by pointer) to int value", + nb::arg("attr").none(), + nb::arg("value").noconvert()) .def( "setAttributeScalar", [](View& self, const Attribute* attr, double value) { return self.setAttributeScalar(attr, value); }, - "Set Attribute (by pointer) to float (C++ double) value") + "Set Attribute (by pointer) to float (C++ double) value", + nb::arg("attr").none(), + nb::arg("value").noconvert()) // String setters .def("setAttributeString", nb::overload_cast(&View::setAttributeString), - "Set Attribute (by index) to string value") + "Set Attribute (by index) to string value", + nb::arg("idx"), + nb::arg("value").noconvert()) .def("setAttributeString", nb::overload_cast(&View::setAttributeString), - "Set Attribute (by name) to string value") + "Set Attribute (by name) to string value", + nb::arg("name"), + nb::arg("value").noconvert()) .def("setAttributeString", nb::overload_cast(&View::setAttributeString), - "Set Attribute (by pointer) to string value") + "Set Attribute (by pointer) to string value", + nb::arg("attr").none(), + nb::arg("value").noconvert()) // Requires conduit::Node information // Scalar getters (Node::ConstValue version) @@ -1214,19 +1414,23 @@ NB_MODULE(_sidre, m_sidre) .def( "getAttributeScalarInt", [](View& self, IndexType idx) { return self.getAttributeScalar(idx); }, - "Return scalar Attribute value (by index) as int") + "Return scalar Attribute value (by index) as int", + nb::arg("idx")) .def( "getAttributeScalarFloat", [](View& self, IndexType idx) { return self.getAttributeScalar(idx); }, - "Return scalar Attribute value (by index) as float (C++ double)") + "Return scalar Attribute value (by index) as float (C++ double)", + nb::arg("idx")) .def( "getAttributeScalarInt", [](View& self, const std::string& name) { return self.getAttributeScalar(name); }, - "Return scalar Attribute value (by name) as int") + "Return scalar Attribute value (by name) as int", + nb::arg("name")) .def( "getAttributeScalarFloat", [](View& self, const std::string& name) { return self.getAttributeScalar(name); }, - "Return scalar Attribute value (by name) as float (C++ double)") + "Return scalar Attribute value (by name) as float (C++ double)", + nb::arg("name")) .def( "getAttributeScalarInt", [](View& self, const Attribute* attr) { return self.getAttributeScalar(attr); }, @@ -1241,13 +1445,16 @@ NB_MODULE(_sidre, m_sidre) // String getters .def("getAttributeString", nb::overload_cast(&View::getAttributeString, nb::const_), - "Return string Attribute value (by index)") + "Return string Attribute value (by index)", + nb::arg("idx")) .def("getAttributeString", nb::overload_cast(&View::getAttributeString, nb::const_), - "Return string Attribute value (by name)") + "Return string Attribute value (by name)", + nb::arg("name")) .def("getAttributeString", nb::overload_cast(&View::getAttributeString, nb::const_), - "Return string Attribute value (by pointer)") + "Return string Attribute value (by pointer)", + nb::arg("attr").none()) // Requires conduit::Node information // Node reference getters @@ -1258,7 +1465,8 @@ NB_MODULE(_sidre, m_sidre) return nodeToNbObject(node); }, nb::rv_policy::reference, - "Return reference to Attribute Node (by index)") + "Return reference to Attribute Node (by index)", + nb::arg("idx")) .def( "getAttributeNodeRef", [](View& self, const std::string& name) { @@ -1266,7 +1474,8 @@ NB_MODULE(_sidre, m_sidre) return nodeToNbObject(node); }, nb::rv_policy::reference, - "Return reference to Attribute Node (by name)") + "Return reference to Attribute Node (by name)", + nb::arg("name")) .def( "getAttributeNodeRef", [](View& self, const Attribute* attr) { @@ -1274,7 +1483,8 @@ NB_MODULE(_sidre, m_sidre) return nodeToNbObject(node); }, nb::rv_policy::reference, - "Return reference to Attribute Node (by pointer)") + "Return reference to Attribute Node (by pointer)", + nb::arg("attr").none()) // Attribute index iteration .def("getFirstValidAttrValueIndex", @@ -1284,7 +1494,8 @@ NB_MODULE(_sidre, m_sidre) .def("getNextValidAttrValueIndex", &View::getNextValidAttrValueIndex, "Return next valid Attribute index for a set Attribute in View object after given index" - "(i.e., smallest index over all Attribute indices larger than given one)"); + "(i.e., smallest index over all Attribute indices larger than given one)", + nb::arg("idx")); // Bindings for the Group class nb::class_(m_sidre, "Group") @@ -1318,44 +1529,56 @@ NB_MODULE(_sidre, m_sidre) .def("hasView", nb::overload_cast(&Group::hasView, nb::const_), - "Return true if Group includes a descendant View with given name or path; else false.") + "Return true if Group includes a descendant View with given name or path; else false.", + nb::arg("path")) .def("hasView", nb::overload_cast(&Group::hasView, nb::const_), - "Return true if this Group owns a View with given index; else false") + "Return true if this Group owns a View with given index; else false", + nb::arg("idx")) .def("hasChildView", &Group::hasChildView, - "Return true if this Group owns a View with given name (not path); else false.") + "Return true if this Group owns a View with given name (not path); else false.", + nb::arg("name")) .def("getViewIndex", &Group::getViewIndex, - "Return index of View with given name owned by this Group object.") + "Return index of View with given name owned by this Group object.", + nb::arg("name")) .def("getViewName", &Group::getViewName, - "Return name of View with given index owned by Group object.") + "Return name of View with given index owned by Group object.", + nb::arg("idx")) .def("getView", nb::overload_cast(&Group::getView, nb::const_), nb::rv_policy::reference_internal, - "Return pointer to const View with given name or path.") + "Return pointer to const View with given name or path.", + nb::arg("path")) .def("getView", nb::overload_cast(&Group::getView, nb::const_), nb::rv_policy::reference_internal, - "Return pointer to non-const View with given index.") + "Return pointer to non-const View with given index.", + nb::arg("idx")) .def("getFirstValidViewIndex", &Group::getFirstValidViewIndex, "Return first valid View index in Group object.") .def("getNextValidViewIndex", &Group::getNextValidViewIndex, - "Return next valid View index in Group object after given index.") + "Return next valid View index in Group object after given index.", + nb::arg("idx")) .def("createView", nb::overload_cast(&Group::createView), nb::rv_policy::reference_internal, - "Create an undescribed (i.e., empty) View object with given name or path in this Group.") + "Create an undescribed (i.e., empty) View object with given name or path in this Group.", + nb::arg("path")) .def("createView", nb::overload_cast(&Group::createView), nb::rv_policy::reference_internal, "Create View object with given name or path in this Group that has a data description " - "with data type and number of elements.") + "with data type and number of elements.", + nb::arg("path"), + nb::arg("type"), + nb::arg("num_elems")) .def( "createViewWithShape", [](Group& self, const std::string& path, TypeID type, int ndims, const nb::ndarray& shape) { @@ -1363,17 +1586,27 @@ NB_MODULE(_sidre, m_sidre) }, nb::rv_policy::reference_internal, "Create View object with given name or path in this Group that has a data description " - "with data type and shape.") + "with data type and shape.", + nb::arg("path"), + nb::arg("type"), + nb::arg("ndims"), + nb::arg("shape")) .def("createView", nb::overload_cast(&Group::createView), nb::rv_policy::reference_internal, "Create an undescribed View object with given name or path in this Group and attach given " - "Buffer to it.") + "Buffer to it.", + nb::arg("path"), + nb::arg("buffer").none()) .def("createView", nb::overload_cast(&Group::createView), nb::rv_policy::reference_internal, "Create View object with given name or path in this Group that has a data description " - "with data type and number of elements and attach given Buffer to it.") + "with data type and number of elements and attach given Buffer to it.", + nb::arg("path"), + nb::arg("type"), + nb::arg("num_elems"), + nb::arg("buffer").none()) .def( "createViewWithShape", [](Group& self, @@ -1386,7 +1619,12 @@ NB_MODULE(_sidre, m_sidre) }, nb::rv_policy::reference_internal, "Create View object with given name or path in this Group that has a data description " - "with data type and shape and attach given Buffer to it.") + "with data type and shape and attach given Buffer to it.", + nb::arg("path"), + nb::arg("type"), + nb::arg("ndims"), + nb::arg("shape"), + nb::arg("buffer").none()) .def( "createView", @@ -1395,7 +1633,9 @@ NB_MODULE(_sidre, m_sidre) pinExternalDataOwner(view, a); return view; }, - nb::rv_policy::reference_internal) + nb::rv_policy::reference_internal, + nb::arg("path"), + nb::arg("external_ptr")) .def( "createView", @@ -1406,7 +1646,11 @@ NB_MODULE(_sidre, m_sidre) }, nb::rv_policy::reference_internal, "Create View object with given name or path in this Group that has a data description " - "with data type and number of elements and attach externally-owned data to it.") + "with data type and number of elements and attach externally-owned data to it.", + nb::arg("path"), + nb::arg("type"), + nb::arg("num_elems"), + nb::arg("external_ptr")) .def( "createViewWithShape", @@ -1422,7 +1666,12 @@ NB_MODULE(_sidre, m_sidre) }, nb::rv_policy::reference_internal, "Create View object with given name or path in this Group that has a data description " - "with data type and shape and attach externally-owned data (numpy array) to it.") + "with data type and shape and attach externally-owned data (numpy array) to it.", + nb::arg("path"), + nb::arg("type"), + nb::arg("ndims"), + nb::arg("shape"), + nb::arg("external_ptr")) .def("createViewAndAllocate", nb::overload_cast(&Group::createViewAndAllocate), nb::rv_policy::reference_internal, @@ -1439,7 +1688,11 @@ NB_MODULE(_sidre, m_sidre) }, nb::rv_policy::reference_internal, "Create View object with given name or path in this Group that has a data description " - "with data type and shape and allocate data for it.") + "with data type and shape and allocate data for it.", + nb::arg("path"), + nb::arg("type"), + nb::arg("ndims"), + nb::arg("shape")) .def("createViewScalar", &Group::createViewScalar, @@ -1472,7 +1725,8 @@ NB_MODULE(_sidre, m_sidre) releaseExternalDataOwner(self.getView(path)); self.destroyView(path); }, - "Destroy View with given name or path owned by this Group, but leave its data intact.") + "Destroy View with given name or path owned by this Group, but leave its data intact.", + nb::arg("path")) .def( "destroyView", [](Group& self, IndexType idx) { @@ -1480,7 +1734,8 @@ NB_MODULE(_sidre, m_sidre) releaseExternalDataOwner(self.getView(idx)); self.destroyView(idx); }, - "Destroy View with given index owned by this Group, but leave its data intact.") + "Destroy View with given index owned by this Group, but leave its data intact.", + nb::arg("idx")) .def( "destroyViewAndData", [](Group& self, const std::string& path) { @@ -1488,7 +1743,8 @@ NB_MODULE(_sidre, m_sidre) releaseExternalDataOwner(self.getView(path)); self.destroyViewAndData(path); }, - "Destroy View with given name or path owned by this Group and deallocate") + "Destroy View with given name or path owned by this Group and deallocate", + nb::arg("path")) .def( "destroyViewAndData", [](Group& self, IndexType idx) { @@ -1496,7 +1752,8 @@ NB_MODULE(_sidre, m_sidre) self.destroyViewAndData(idx); }, "Destroy View with given index owned by this Group and deallocate its data if it's the " - "only View associated with that data.") + "only View associated with that data.", + nb::arg("idx")) .def( "destroyViewsAndData", [](Group& self) { @@ -1509,7 +1766,8 @@ NB_MODULE(_sidre, m_sidre) .def("moveView", &Group::moveView, nb::rv_policy::reference_internal, - "Remove given View object from its owning Group and move it to this Group.") + "Remove given View object from its owning Group and move it to this Group.", + nb::arg("view")) .def( "copyView", [](Group& self, View* view) { @@ -1523,31 +1781,39 @@ NB_MODULE(_sidre, m_sidre) return copy; }, nb::rv_policy::reference_internal, - "Create a (shallow) copy of given View object and add it to this Group.") + "Create a (shallow) copy of given View object and add it to this Group.", + nb::arg("view")) .def("hasGroup", nb::overload_cast(&Group::hasGroup, nb::const_), - "Return true if this Group has a descendant Group with given name or path; else false.") + "Return true if this Group has a descendant Group with given name or path; else false.", + nb::arg("path")) .def("hasGroup", nb::overload_cast(&Group::hasGroup, nb::const_), - "Return true if Group has an immediate child Group with given index; else false.") + "Return true if Group has an immediate child Group with given index; else false.", + nb::arg("idx")) .def("hasChildGroup", &Group::hasChildGroup, - "Return true if this Group has a child Group with given name; else false.") + "Return true if this Group has a child Group with given name; else false.", + nb::arg("name")) .def("getGroupIndex", &Group::getGroupIndex, - "Return the index of immediate child Group with given name.") + "Return the index of immediate child Group with given name.", + nb::arg("name")) .def("getGroupName", &Group::getGroupName, - "Return the name of immediate child Group with given index.") + "Return the name of immediate child Group with given index.", + nb::arg("idx")) .def("getGroup", nb::overload_cast(&Group::getGroup), nb::rv_policy::reference_internal, - "Return pointer to non-const child Group with given name or path.") + "Return pointer to non-const child Group with given name or path.", + nb::arg("path")) .def("getGroup", nb::overload_cast(&Group::getGroup), nb::rv_policy::reference_internal, - "Return pointer to non-const immediate child Group with given index.") + "Return pointer to non-const immediate child Group with given index.", + nb::arg("idx")) .def("views", nb::overload_cast<>(&Group::views), nb::keep_alive<0, 1>(), @@ -1561,7 +1827,8 @@ NB_MODULE(_sidre, m_sidre) "Return first valid child Group index (i.e., smallest index over all child Groups).") .def("getNextValidGroupIndex", &Group::getNextValidGroupIndex, - "Return next valid child Group index after given index.") + "Return next valid child Group index after given index.", + nb::arg("idx")) .def("createGroup", &Group::createGroup, nb::rv_policy::reference_internal, @@ -1581,7 +1848,8 @@ NB_MODULE(_sidre, m_sidre) releaseExternalDataOwners(self.getGroup(path)); self.destroyGroup(path); }, - "Destroy child Group in this Group with given name or path.") + "Destroy child Group in this Group with given name or path.", + nb::arg("path")) .def( "destroyGroup", [](Group& self, IndexType idx) { @@ -1589,7 +1857,8 @@ NB_MODULE(_sidre, m_sidre) releaseExternalDataOwners(self.getGroup(idx)); self.destroyGroup(idx); }, - "Destroy child Group within this Group with given index.") + "Destroy child Group within this Group with given index.", + nb::arg("idx")) .def( "destroyGroupAndData", [](Group& self, const std::string& path) { @@ -1598,7 +1867,8 @@ NB_MODULE(_sidre, m_sidre) self.destroyGroupAndData(path); }, "Destroy child Group at the given path, and destroy data that is " - "not shared elsewhere.") + "not shared elsewhere.", + nb::arg("path")) .def( "destroyGroupAndData", [](Group& self, IndexType idx) { @@ -1606,7 +1876,8 @@ NB_MODULE(_sidre, m_sidre) self.destroyGroupAndData(idx); }, "Destroy child Group with the given index, and destroy data that " - "is not shared elsewhere.") + "is not shared elsewhere.", + nb::arg("idx")) .def( "destroyGroupsAndData", [](Group& self) { @@ -1639,7 +1910,8 @@ NB_MODULE(_sidre, m_sidre) .def("moveGroup", &Group::moveGroup, nb::rv_policy::reference_internal, - "Remove given Group object from its parent Group and make it a child of this Group.") + "Remove given Group object from its parent Group and make it a child of this Group.", + nb::arg("group")) .def( "copyGroup", [](Group& self, Group* group) { @@ -1653,7 +1925,8 @@ NB_MODULE(_sidre, m_sidre) }, nb::rv_policy::reference_internal, "Create a (shallow) copy of Group hierarchy rooted at given " - "Group and make the copy a child of this Group.") + "Group and make the copy a child of this Group.", + nb::arg("group")) .def("deepCopyGroup", &Group::deepCopyGroup, nb::rv_policy::reference_internal, @@ -1700,11 +1973,12 @@ NB_MODULE(_sidre, m_sidre) .def("loadExternalData", nb::overload_cast(&Group::loadExternalData), - "Load data into the Group's external views from a file.") + "Load data into the Group's external views from a file.", + nb::arg("path")) .def_static("getDefaultIOProtocol", &Group::getDefaultIOProtocol, "Return the default I/O protocol for this Axom build.") - .def("rename", &Group::rename, "Change the name of this Group."); + .def("rename", &Group::rename, "Change the name of this Group.", nb::arg("new_name")); // Bindings for the Attribute class nb::class_(m_sidre, "Attribute") @@ -1721,7 +1995,8 @@ NB_MODULE(_sidre, m_sidre) nb::arg("value").noconvert()) .def("setDefaultString", &Attribute::setDefaultString, - "Set default value of Attribute as string. Return true if successfully changed.") + "Set default value of Attribute as string. Return true if successfully changed.", + nb::arg("value").noconvert()) .def( "getDefaultNodeRef", @@ -1827,7 +2102,8 @@ NB_MODULE(_sidre, m_sidre) nb::arg("root_file")) .def_static("correspondingRelayProtocol", &IOManager::correspondingRelayProtocol, - "Finds conduit relay protocol corresponding to a sidre protocol."); + "Finds conduit relay protocol corresponding to a sidre protocol.", + nb::arg("sidre_protocol")); #endif } diff --git a/src/axom/sidre/tests/sidre_attribute_Py.py b/src/axom/sidre/tests/sidre_attribute_Py.py index ac92fc0fd5..083d0af1bd 100644 --- a/src/axom/sidre/tests/sidre_attribute_Py.py +++ b/src/axom/sidre/tests/sidre_attribute_Py.py @@ -6,6 +6,7 @@ import axom.sidre as sidre import numpy as np +import pytest import conduit # Global attribute values, used by multiple tests @@ -945,3 +946,62 @@ def test_save_load_group_with_attributes_same_ds(): assert gr.getView("scalar2").getAttributeString(g_name_color) == g_color_red assert gr.getView("scalar3").hasAttributeValue(g_name_color) assert gr.getView("scalar3").getAttributeString(g_name_color) == g_color_blue + + +# --------------------------------------------------------------------------- +# Scalar setters require an exact python int or float +# --------------------------------------------------------------------------- +# The int and float overloads of the scalar setters are bound with nb::arg("value").noconvert(), +# so nanobind skips its converting overload pass, so numpy floats don't silently get bound +# to the int and truncated. As a consequence, numpy scalars must be converted by the caller, +# e.g. float(x) or x.item(). +def test_setAttributeScalar_requires_exact_python_scalar_types(): + ds = sidre.DataStore() + ds.createAttributeScalar(g_name_dump, g_dump_no) + view = ds.getRoot().createViewScalar("scalar", 0) + + # Exact python types are accepted. + assert view.setAttributeScalar(g_name_dump, 1) + assert view.getAttributeScalarInt(g_name_dump) == 1 + + # numpy scalars, 0-d arrays, bool and str are rejected rather than converted. + for rejected in (np.int32(1), np.int64(1), np.float32(1.0), np.float64(1.0), np.array(1), True, + "1"): + with pytest.raises(TypeError): + view.setAttributeScalar(g_name_dump, rejected) + + # The value is unchanged by the rejected calls. + assert view.getAttributeScalarInt(g_name_dump) == 1 + + # The documented conversion at the call site works. + assert view.setAttributeScalar(g_name_dump, int(np.int64(7))) + assert view.getAttributeScalarInt(g_name_dump) == 7 + + +def test_noconvert_prevents_silent_float_to_int_truncation(): + # This is what the noconvert annotations buy. With conversion enabled, + # np.float32(3.5) binds to the int overload and stores 3. + ds = sidre.DataStore() + ds.createAttributeScalar(g_name_size, g_size_small) + view = ds.getRoot().createViewScalar("scalar", 0) + + with pytest.raises(TypeError): + view.setAttributeScalar(g_name_size, np.float32(3.5)) + + # Converting explicitly keeps the fractional part. + assert view.setAttributeScalar(g_name_size, float(np.float32(3.5))) + assert view.getAttributeScalarFloat(g_name_size) == 3.5 + + +def test_setScalar_requires_exact_python_scalar_types(): + # The same contract on View.setScalar, which has carried noconvert since + # before the attribute setters did. + ds = sidre.DataStore() + view = ds.getRoot().createViewScalar("scalar", 0) + + assert view.setScalar(5) is not None + assert view.getDataInt() == 5 + + for rejected in (np.int64(5), np.float64(5.0), np.array(5), True): + with pytest.raises(TypeError): + view.setScalar(rejected) diff --git a/src/axom/sidre/tests/sidre_lifetime_Py.py b/src/axom/sidre/tests/sidre_lifetime_Py.py index 2196179523..6b7417a602 100644 --- a/src/axom/sidre/tests/sidre_lifetime_Py.py +++ b/src/axom/sidre/tests/sidre_lifetime_Py.py @@ -891,6 +891,215 @@ def test_concurrent_datastores_registry_isolation(): assert ref2() is None, "DS2 external data not collected after destroyView" +# --------------------------------------------------------------------------- +# External views onto sidre-owned storage must not pin their own DataStore +# --------------------------------------------------------------------------- +# The binding pins the numpy owner of an external view so a dropped temporary +# cannot leave sidre holding a dangling pointer. Storage that sidre already owns +# must be exempt: pinning it forms a cycle the registry cannot break (pin -> array +# -> sidre wrapper -> DataStore python object, whose collection is what releases +# the pin), retaining the DataStore, Group, View and Buffer for the life of the process. +def test_opaque_view_onto_sidre_storage_does_not_retain_datastore(): + ds = sidre.DataStore() + root = ds.getRoot() + field = root.createViewAndAllocate("field", sidre.TypeID.FLOAT64_ID, 20) + data = field.getBuffer().getDataArray() + + ref = weakref.ref(ds) + root.createView("aliased", data) # undescribed/opaque overload + + del data, field, root, ds + _force_gc() + + assert ref() is None, "DataStore retained by a pin onto sidre-owned storage" + + +def test_described_external_view_onto_sidre_storage_does_not_retain_datastore(): + ds = sidre.DataStore() + root = ds.getRoot() + field = root.createViewAndAllocate("field", sidre.TypeID.FLOAT64_ID, 20) + data = field.getBuffer().getDataArray() + + ref = weakref.ref(ds) + root.createView("aliased", sidre.TypeID.FLOAT64_ID, 20, data) + + del data, field, root, ds + _force_gc() + + assert ref() is None, "DataStore retained by a pin onto sidre-owned storage" + + +def test_external_view_onto_sidre_storage_still_reads_correctly(): + # The exemption removes the pin, not the aliasing: + # the view must still read the buffer it points into. + ds = sidre.DataStore() + root = ds.getRoot() + field = root.createViewAndAllocate("field", sidre.TypeID.FLOAT64_ID, 8) + data = field.getBuffer().getDataArray() + data[:] = np.arange(8) + 1.0 + + view = root.createView("aliased", sidre.TypeID.FLOAT64_ID, 8, data) + del data + _force_gc() + + assert view.getDataArray()[0] == 1.0 + assert view.getDataArray()[7] == 8.0 + + +# The exemption lives in one place (pinExternalDataOwner), so every entry point +# that pins inherits it. Cover the two that createView does not reach, and the +# boundary the exemption must not cross. +def test_set_external_data_onto_sidre_storage_does_not_retain_datastore(): + ds = sidre.DataStore() + root = ds.getRoot() + field = root.createViewAndAllocate("field", sidre.TypeID.FLOAT64_ID, 8) + data = field.getBuffer().getDataArray() + + target = root.createView("aliased") + target.setExternalData(sidre.TypeID.FLOAT64_ID, 8, data) + + ref = weakref.ref(ds) + del target, data, field, root, ds + _force_gc() + + assert ref() is None, "DataStore retained by a setExternalData pin onto its own storage" + + +def test_copied_view_onto_sidre_storage_does_not_retain_datastore(): + ds = sidre.DataStore() + root = ds.getRoot() + field = root.createViewAndAllocate("field", sidre.TypeID.FLOAT64_ID, 8) + data = field.getBuffer().getDataArray() + source = root.createView("aliased", sidre.TypeID.FLOAT64_ID, 8, data) + + # copyView re-pins the destination from the source's pin; an exempt source has + # no pin to copy, so the destination must not acquire one either. + root.createGroup("copy_target").copyView(source) + + ref = weakref.ref(ds) + del source, data, field, root, ds + _force_gc() + + assert ref() is None, "DataStore retained by a copied view's pin onto its own storage" + + +def test_external_view_onto_another_datastores_storage_is_still_pinned(): + # The exemption is per-DataStore: a view in the `consumer` DataStore pointing at storage + # owned by the `donor` DataStore is not exempt and must still be pinned. + # The observable consequence is that the pin keeps the donor alive. + donor = sidre.DataStore() + donor_field = donor.getRoot().createViewAndAllocate("field", sidre.TypeID.FLOAT64_ID, 8) + data = donor_field.getBuffer().getDataArray() + data[:] = np.arange(8) + 1.0 + + consumer = sidre.DataStore() + view = consumer.getRoot().createView("aliased", sidre.TypeID.FLOAT64_ID, 8, data) + + donor_ref = weakref.ref(donor) + del data, donor_field, donor + _force_gc() + + assert donor_ref() is not None, "aliased donor storage was not pinned by the consuming view" + assert view.getDataArray()[0] == 1.0 + assert view.getDataArray()[7] == 8.0 + + # Pinning across DataStores must not make the *consumer* immortal: + # its pin references the donor, not itself, so collecting it releases the donor too. + consumer_ref = weakref.ref(consumer) + del view, consumer + _force_gc() + + assert consumer_ref() is None, "consumer DataStore retained by its own external-data pin" + assert donor_ref() is None, "donor storage still pinned after the consuming view went away" + + +# --------------------------------------------------------------------------- +# Aliasing an already-pinned external view must not pin a sidre wrapper +# --------------------------------------------------------------------------- +# Storage behind an external view is owned by Python, not sidre, so a view onto +# it does need a pin -- but the array handed in may be +# `external_view.getDataArray()`, which is owned by that View's python wrapper. +# Pinning that array recreates the cycle the buffer exemption avoids +# (pin -> array -> View wrapper -> DataStore python object, whose collection is +# what releases the pin). The pin must be redirected to the original numpy owner. +def test_view_aliasing_a_pinned_external_view_does_not_retain_datastore(): + ds = sidre.DataStore() + root = ds.getRoot() + source_data = np.arange(8, dtype=np.float64) + 1.0 + external = root.createView("external", sidre.TypeID.FLOAT64_ID, 8, source_data) + + # Owned by `external`'s python wrapper, not by source_data. + aliased_data = external.getDataArray() + root.createView("aliased", sidre.TypeID.FLOAT64_ID, 8, aliased_data) + + ref = weakref.ref(ds) + del aliased_data, external, root, ds + _force_gc() + + assert ref() is None, "DataStore retained by a pin onto its own external view's storage" + + +def test_view_aliasing_a_pinned_external_view_still_pins_the_numpy_owner(): + # The redirect must not drop the pin. Destroy the source view so *its* pin is + # gone, leaving the aliasing view's redirected pin as the only thing keeping + # the numpy storage alive. A fix that simply skipped the pin (the way the + # sidre-owned case does) would let the array be collected here and leave the + # aliasing view pointing at freed memory. + ds = sidre.DataStore() + root = ds.getRoot() + source_data = np.arange(8, dtype=np.float64) + 1.0 + external = root.createView("external", sidre.TypeID.FLOAT64_ID, 8, source_data) + + aliased = root.createView("aliased", sidre.TypeID.FLOAT64_ID, 8, external.getDataArray()) + + array_ref = weakref.ref(source_data) + del external + root.destroyView("external") # releases the source view's own pin + del source_data + _force_gc() + + assert array_ref() is not None, "numpy owner was not pinned by the aliasing view" + assert aliased.getDataArray()[0] == 1.0 + assert aliased.getDataArray()[7] == 8.0 + + # Drop everything before returning: leaving a live DataStore (and its pin) + # in this frame perturbs later tests in this module, which assert on + # collection of their own DataStores. + del aliased, root, ds + _force_gc() + + +def test_setExternalData_aliasing_a_pinned_external_view_does_not_retain_datastore(): + # Same redirect, reached through setExternalData rather than createView. + # Structured like the createView case above so it discriminates the redirect + # from a fix that merely skips the pin: the source view is destroyed, so the + # redirected pin is the only remaining reference to the numpy storage. + ds = sidre.DataStore() + root = ds.getRoot() + source_data = np.arange(8, dtype=np.float64) + 1.0 + external = root.createView("external", sidre.TypeID.FLOAT64_ID, 8, source_data) + + target = root.createView("target") + target.setExternalData(sidre.TypeID.FLOAT64_ID, 8, external.getDataArray()) + + array_ref = weakref.ref(source_data) + del external + root.destroyView("external") # releases the source view's own pin + del source_data + _force_gc() + + assert array_ref() is not None, "numpy owner was not pinned by the aliasing view" + assert target.getDataArray()[0] == 1.0 + assert target.getDataArray()[7] == 8.0 + + ref = weakref.ref(ds) + del target, root, ds + _force_gc() + + assert ref() is None, ( + "DataStore retained by a setExternalData pin onto its own external storage") + + if __name__ == "__main__": import sys diff --git a/src/cmake/axom-config.cmake.in b/src/cmake/axom-config.cmake.in index 66987f6603..37fdaaad98 100644 --- a/src/cmake/axom-config.cmake.in +++ b/src/cmake/axom-config.cmake.in @@ -30,6 +30,7 @@ if(NOT AXOM_FOUND) #---------------------------------------------------------------------------- # Language features + set(AXOM_CXX_STANDARD "@CMAKE_CXX_STANDARD@") set(AXOM_ENABLE_FORTRAN "@ENABLE_FORTRAN@") set(AXOM_USE_CUDA "@AXOM_USE_CUDA@") set(AXOM_USE_HIP "@AXOM_USE_HIP@") @@ -151,9 +152,13 @@ if(NOT AXOM_FOUND) # conduit if(AXOM_USE_CONDUIT) set(AXOM_CONDUIT_DIR "@CONDUIT_DIR@") + set(AXOM_CONDUIT_PYTHON_MODULE_DIR "@CONDUIT_PYTHON_MODULE_DIR@") if(NOT CONDUIT_DIR) set(CONDUIT_DIR ${AXOM_CONDUIT_DIR}) endif() + if(NOT CONDUIT_PYTHON_MODULE_DIR AND AXOM_CONDUIT_PYTHON_MODULE_DIR) + set(CONDUIT_PYTHON_MODULE_DIR "${AXOM_CONDUIT_PYTHON_MODULE_DIR}") + endif() # Load mpi targets because we require the optional Conduit mpi targets if(AXOM_USE_MPI) diff --git a/src/cmake/thirdparty/SetupAxomThirdParty.cmake b/src/cmake/thirdparty/SetupAxomThirdParty.cmake index 6bc358c31f..9c6a4e304a 100644 --- a/src/cmake/thirdparty/SetupAxomThirdParty.cmake +++ b/src/cmake/thirdparty/SetupAxomThirdParty.cmake @@ -135,6 +135,20 @@ if (CONDUIT_DIR) set(CONDUIT_FOUND TRUE) blt_convert_to_system_includes(TARGET conduit::conduit) + + # Resolve CONDUIT_PYTHON_MODULE_DIR to an absolute path + # Preserve user-supplied cache values if present over the one from conduit's install + get_property(_axom_conduit_py_dir_cache + CACHE CONDUIT_PYTHON_MODULE_DIR PROPERTY VALUE) + if(_axom_conduit_py_dir_cache) + set(CONDUIT_PYTHON_MODULE_DIR "${_axom_conduit_py_dir_cache}") + endif() + unset(_axom_conduit_py_dir_cache) + + if(CONDUIT_PYTHON_MODULE_DIR AND NOT IS_ABSOLUTE "${CONDUIT_PYTHON_MODULE_DIR}") + get_filename_component(CONDUIT_PYTHON_MODULE_DIR + "${CONDUIT_DIR}/${CONDUIT_PYTHON_MODULE_DIR}" ABSOLUTE) + endif() else() message(STATUS "Conduit support is OFF") endif() diff --git a/src/python/CMakeLists.txt b/src/python/CMakeLists.txt new file mode 100644 index 0000000000..51045c141f --- /dev/null +++ b/src/python/CMakeLists.txt @@ -0,0 +1,190 @@ +# Copyright (c) Lawrence Livermore National Security, LLC and other +# Axom Project Contributors. See top-level LICENSE and COPYRIGHT +# files for dates and other details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +#------------------------------------------------------------------------------ +# Thin, binding-only build of Axom's Python extension module(s). +# +# This project compiles Axom's Python bindings against an installed Axom. +# It does not build Axom or its third-party libraries. +#------------------------------------------------------------------------------ + +cmake_minimum_required(VERSION 3.21) +project(axom_python LANGUAGES C CXX) + +# Enable with both build-time flags; see src/python/README.md: +# -C cmake.define.AXOM_PYTHON_STABLE_ABI=ON -C wheel.py-api=cp312 +option(AXOM_PYTHON_STABLE_ABI + "Build the extension against Python's stable ABI (abi3); needs Python >= 3.12" OFF) + +if(AXOM_PYTHON_STABLE_ABI AND CMAKE_VERSION VERSION_LESS 3.26) + message(FATAL_ERROR + "AXOM_PYTHON_STABLE_ABI=ON requires CMake >= 3.26 because " + "FindPython's Development.SABIModule component is needed " + "to locate Python stable ABI development artifacts. " + "Current CMake is ${CMAKE_VERSION}. " + "Build with CMake >= 3.26 or leave AXOM_PYTHON_STABLE_ABI off.") +endif() + +if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.26) + find_package(Python 3.10 REQUIRED + COMPONENTS Interpreter Development.Module + OPTIONAL_COMPONENTS Development.SABIModule) +else() + find_package(Python 3.10 REQUIRED + COMPONENTS Interpreter Development.Module) +endif() + +if(AXOM_PYTHON_STABLE_ABI AND NOT Python_Development.SABIModule_FOUND) + message(FATAL_ERROR + "AXOM_PYTHON_STABLE_ABI=ON requires CMake >= 3.26 and Python's " + "Development.SABIModule component (CPython >= 3.12), " + "which was not found for ${Python_EXECUTABLE} (version ${Python_VERSION}). " + "Build with a 3.12+ interpreter that provides stable ABI development artifacts, " + "or leave AXOM_PYTHON_STABLE_ABI off.") +endif() + +if(DEFINED AXOM_DIR AND NOT DEFINED axom_DIR) + set(axom_DIR "${AXOM_DIR}" CACHE PATH "Axom CMake package directory") +endif() + +find_package(axom CONFIG REQUIRED) +find_package(nanobind CONFIG REQUIRED) # supplied via build-system.requires + +# Compile the bindings with the C++ standard of the Axom install. +if(NOT AXOM_CXX_STANDARD) + set(AXOM_CXX_STANDARD 20) +endif() + +if(NOT DEFINED CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD ${AXOM_CXX_STANDARD}) +elseif(CMAKE_CXX_STANDARD LESS AXOM_CXX_STANDARD) + message(FATAL_ERROR + "CMAKE_CXX_STANDARD is C++${CMAKE_CXX_STANDARD}, " + "but the Axom install at '${axom_DIR}' was built with C++${AXOM_CXX_STANDARD}.") +endif() + +set(_axom_py_conduit_cmake_dir "${Conduit_DIR}") +if(NOT _axom_py_conduit_cmake_dir AND AXOM_CONDUIT_DIR) + set(_axom_py_conduit_cmake_dir "${AXOM_CONDUIT_DIR}/lib/cmake/conduit") +endif() + +# Conduit's Python package directory -- needs to be absolute +# +# NOTE: Renamed from CONDUIT_PYTHON_MODULE_DIR to avoid collision: +# find_package(axom) pulls in ConduitConfig.cmake, which sets +# CONDUIT_PYTHON_MODULE_DIR with plain set() and would overwrite +# any value the user passed. AXOM_PYTHON_CONDUIT_MODULE_DIR is +# distinct and not shadowed by Conduit's config. +set(AXOM_PYTHON_CONDUIT_MODULE_DIR "" CACHE PATH + "Directory holding the same-build conduit Python package; overrides the value recorded by the Axom install") + +if(AXOM_PYTHON_CONDUIT_MODULE_DIR) # use explicit override, if provided + set(_axom_py_conduit_python_module_dir "${AXOM_PYTHON_CONDUIT_MODULE_DIR}") +elseif(AXOM_CONDUIT_PYTHON_MODULE_DIR) # else, use value from Axom's export + set(_axom_py_conduit_python_module_dir "${AXOM_CONDUIT_PYTHON_MODULE_DIR}") +else() # and fallback to conduit's path + set(_axom_py_conduit_python_module_dir "${CONDUIT_PYTHON_MODULE_DIR}") +endif() + +if(_axom_py_conduit_python_module_dir + AND NOT IS_ABSOLUTE "${_axom_py_conduit_python_module_dir}") + get_filename_component(_axom_py_conduit_python_module_dir + "${AXOM_CONDUIT_DIR}/${_axom_py_conduit_python_module_dir}" ABSOLUTE) +endif() + +if(NOT _axom_py_conduit_python_module_dir + OR NOT EXISTS "${_axom_py_conduit_python_module_dir}/conduit") + message(FATAL_ERROR + "Could not find Conduit's Python package for this Axom install.\n" + " Looked for a directory containing a 'conduit' package at: " + "'${_axom_py_conduit_python_module_dir}'\n" + " AXOM_CONDUIT_DIR = ${AXOM_CONDUIT_DIR}\n" + " AXOM_CONDUIT_PYTHON_MODULE_DIR = ${AXOM_CONDUIT_PYTHON_MODULE_DIR}\n" + "Either rebuild Axom against a Conduit configured with Python support, or " + "pass the path explicitly:\n" + " -C cmake.define.AXOM_PYTHON_CONDUIT_MODULE_DIR=") +endif() + +# Keep the wheel metadata version aligned with the Axom install it links. +if(DEFINED SKBUILD_PROJECT_VERSION AND DEFINED AXOM_VERSION_MAJOR) + set(_axom_installed_version + "${AXOM_VERSION_MAJOR}.${AXOM_VERSION_MINOR}.${AXOM_VERSION_PATCH}") + if(NOT SKBUILD_PROJECT_VERSION VERSION_EQUAL _axom_installed_version) + message(FATAL_ERROR + "Axom version mismatch: the wheel's metadata version is " + "${SKBUILD_PROJECT_VERSION} (from src/cmake/AxomVersion.cmake in this " + "source tree) but the Axom install it would link against is " + "${_axom_installed_version} (AXOM_DIR=${axom_DIR}). Build the wheel from " + "the source tree that produced the install, or point AXOM_DIR at an " + "install built from this source tree.") + endif() +endif() + +#------------------------------------------------------------------------------ +# Binding sources. The translation unit lives with its component, not in this project directory. +#------------------------------------------------------------------------------ +set(_sidre_binding_sources + "${CMAKE_CURRENT_SOURCE_DIR}/../axom/sidre/nanobind_sidre.cpp") + +# Match the in-tree Sidre binding domain so future Axom modules can share bound +# C++ types when built together. +set(_axom_nb_module_args NB_DOMAIN axom) +if(AXOM_PYTHON_STABLE_ABI) + list(APPEND _axom_nb_module_args STABLE_ABI) +endif() +nanobind_add_module(_sidre ${_axom_nb_module_args} ${_sidre_binding_sources}) + +target_link_libraries(_sidre PRIVATE axom::sidre conduit::conduit_python) + +set_target_properties(_sidre PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE) + +#------------------------------------------------------------------------------ +# Generated runtime/development helpers tied to the Axom/Conduit install. +#------------------------------------------------------------------------------ +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cmake/axom-python-host-config.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/axom-python-host-config.cmake" + @ONLY) +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cmake/axom-python-env.sh.in" + "${CMAKE_CURRENT_BINARY_DIR}/axom-python-env.sh" + @ONLY) +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/axom-python-host-config.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/axom-python-env.sh" + DESTINATION axom/share) + +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cmake/conduit.pth.in" + "${CMAKE_CURRENT_BINARY_DIR}/conduit.pth" + @ONLY) +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/conduit.pth" + DESTINATION ".") + +#------------------------------------------------------------------------------ +# TODO: +# - Check that we can generate local bindings for HIP/CUDA +#------------------------------------------------------------------------------ + +#------------------------------------------------------------------------------ +# nanobind_add_stub imports the module ('import _sidre') to introspect it. +# Turn this off on hosts where the build-time import is problematic: +# -C cmake.define.AXOM_PYTHON_GENERATE_STUB=OFF +#------------------------------------------------------------------------------ +option(AXOM_PYTHON_GENERATE_STUB + "Generate the _sidre.pyi type stub at build time (imports the module)" ON) + +if(AXOM_PYTHON_GENERATE_STUB) + nanobind_add_stub( + _sidre_stub + MODULE _sidre + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/_sidre.pyi" + PYTHON_PATH $ + DEPENDS _sidre) + + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/_sidre.pyi" + DESTINATION axom/sidre) +endif() + +#------------------------------------------------------------------------------ +# Pure-Python package files ship via [tool.scikit-build] wheel.packages. +#------------------------------------------------------------------------------ +install(TARGETS _sidre LIBRARY DESTINATION axom/sidre) diff --git a/src/python/README.md b/src/python/README.md index 92125f71c9..c92890cff7 100644 --- a/src/python/README.md +++ b/src/python/README.md @@ -13,14 +13,38 @@ It is consumed by two independent build paths that must produce the same on-disk 1. **The CMake build (in tree).** When Axom is configured with a component's Python bindings enabled (currently Sidre), the build stages this tree into the build directory and installs it into a `site-packages`-shaped prefix. - See `src/axom/sidre/CMakeLists.txt`: it copies the files below into `${PROJECT_BINARY_DIR}/python/` + See `src/axom/sidre/CMakeLists.txt`: it copies the files below into `${PROJECT_BINARY_DIR}/python/` (so the build tree is import-ready) and installs them under `AXOM_PYTHON_MODULE_INSTALL_PREFIX`. The compiled extension (`_sidre`) and its type stub are emitted into this layout by the build; they are not checked in. -2. **[planned] The pip/uv wheel (out of tree).** A thin, binding-only wheel built with scikit-build-core - will treat this directory as its package root (`wheel.packages = ["src/axom"]` in a sibling `pyproject.toml`), - compiling the binding translation unit against an already-installed Axom. +2. **The pip/uv wheel (out of tree).** A thin, binding-only wheel built with scikit-build-core + (the `pyproject.toml` and `CMakeLists.txt` beside this file). It treats this directory as its + package root (`wheel.packages = ["src/axom"]`) and compiles the binding translation unit against + an already-installed Axom. +This file discusses contributor-facing concerns. Installing and using the bindings is documented +in the Sidre user guide's "Python interface" page (`src/axom/sidre/docs/sphinx/python_interface.rst`). + +## Build paths at a glance + +Both build paths install the same package layout and should expose the same Python API: + +- the **same** binding translation unit (`src/axom/sidre/nanobind_sidre.cpp`) +- under the **same** nanobind domain (`NB_DOMAIN axom`) +- with the **same** pure-Python tree from this directory. + +They differ in where the Axom C++ libraries come from: + +- **In-tree CMake build.** Axom's normal CMake build compiles the C++ libraries, + builds `_sidre` in the same build tree, stages the package under `/python/`, + and installs it under `AXOM_PYTHON_MODULE_INSTALL_PREFIX`. +- **Thin pip/uv wheel.** The scikit-build-core project in this directory consumes + an already-installed Axom via `find_package(axom CONFIG REQUIRED)` and compiles + only the Python binding module against that install. + +The wheel deliberately does not build Axom, Conduit, HDF5, RAJA, Umpire, MPI, or other TPLs. +Those come from the CMake/spack side. Conduit is also not listed as a Python dependency +because `axom.sidre` must import the Python module from the same Conduit build that Axom links. ## Layout @@ -29,9 +53,12 @@ This is a standard "src layout" Python project root: ``` src/python/ README.md <- this file + pyproject.toml <- scikit-build-core project for the pip/uv wheel + CMakeLists.txt <- wheel build: finds an installed Axom, builds the extension src/ axom/ <- the 'axom' regular package __init__.py <- top-level package metadata + config.py <- locates the wheel-generated helpers (axom-python-config) py.typed <- PEP 561 marker (typed package) sidre/ __init__.py <- re-exports the compiled 'axom.sidre._sidre' @@ -42,6 +69,11 @@ src/python/ Parenthesized entries are build products and are intentionally not in the repository. +Both build paths install this tree. Only the wheel build additionally generates +`axom/share/axom-python-host-config.cmake` and `axom/share/axom-python-env.sh`; +on a CMake installation `axom.config.has_wheel_config()` returns `False` and the +path accessors raise `FileNotFoundError` with that explanation. + Each bound Axom component installs as a submodule of the `axom` package (`axom.sidre`, and later `axom.quest`, `axom.primal`, ...). A submodule is importable only when its component was enabled in the underlying Axom build. @@ -55,9 +87,128 @@ A submodule is importable only when its component was enabled in the underlying generated artifacts (the `.so` and `.pyi` are produced by the build), and tests/examples (those live under the component, e.g. `src/axom/sidre/tests/*_Py.py`). -## Notes +## Wheel build reference + +The wheel compiles Axom's Python binding against an existing Axom install. +It is specific to that install and host-config; it is not repaired with `auditwheel` +and is not intended for PyPI. + +Use an absolute `AXOM_DIR` pointing at the directory containing `axom-config.cmake`, +normally `$AXOM_INSTALL/lib/cmake`: + +```bash +uv build --wheel -C cmake.define.AXOM_DIR="$AXOM_INSTALL/lib/cmake" src/python +``` + +The underlying CMake package variable is `axom_DIR`. +`AXOM_DIR` is accepted as an Axom-conventional alias. +Do not use `CMAKE_PREFIX_PATH` for `uv build` or `uv pip install` since scikit-build-core +uses it internally for the isolated build environment. + +Conduit and its Python package path are found through `axom-config.cmake` in the normal case. +Add `Conduit_DIR` only if Axom's recorded Conduit package path no longer resolves: + +```bash +uv build --wheel \ + -C cmake.define.AXOM_DIR="$AXOM_INSTALL/lib/cmake" \ + -C cmake.define.Conduit_DIR="$CONDUIT_INSTALL/lib/cmake/conduit" \ + src/python +``` + +Add `AXOM_PYTHON_CONDUIT_MODULE_DIR` only if Axom's recorded Conduit Python package path is +missing or stale: + +```bash +uv build --wheel \ + -C cmake.define.AXOM_DIR="$AXOM_INSTALL/lib/cmake" \ + -C cmake.define.AXOM_PYTHON_CONDUIT_MODULE_DIR="$CONDUIT_INSTALL/python-modules" \ + src/python +``` + +Note the deliberately distinct name: `CONDUIT_PYTHON_MODULE_DIR` cannot be used here. +`find_package(axom)` pulls in `ConduitConfig.cmake`, which sets that variable with a plain +`set()` and so overwrites whatever the caller passed. + +When building against a host-config, pass the same cache script used for the +Axom install instead of duplicating compiler and MPI settings one variable at a +time: + +```bash +uv build --wheel \ + -C cmake.args=-C \ + -C cmake.args=/absolute/path/to/host-config.cmake \ + -C cmake.define.AXOM_DIR="$AXOM_INSTALL/lib/cmake" \ + src/python +``` + +Build from the source tree that produced the install. The build compares the +wheel metadata version with the installed Axom version and fails if they differ. + +The wheel also installs development helpers, which report their own paths: + +```bash +axom-python-config --host-config # path to axom/share/axom-python-host-config.cmake +axom-python-config --env-script # path to axom/share/axom-python-env.sh +``` + +The host-config seeds a downstream CMake project with the same Axom, Conduit, +compiler, `ENABLE_MPI`, MPI wrapper and Python settings the wheel used; the env script +exports the subset of those that CMake reads from the environment. Both are generated only +by this wheel build, not by the in-tree CMake install. See the "pip / uv wheel" +section of the Sidre user guide for the usage examples. + +### Developer loop (editable, rebuild-on-import) + +nanobind's recommended editable flow rebuilds the extension automatically when +you re-import it after editing the binding source. Rebuild-on-import is a +scikit-build-core *experimental* feature (`editable.rebuild=true`) and may change; +if it misbehaves, reinstall the editable wheel to force a rebuild: + +```bash +uv pip install nanobind 'scikit-build-core[pyproject]' +uv pip install -e src/python --no-build-isolation \ + -C cmake.define.AXOM_DIR="$AXOM_INSTALL/lib/cmake" \ + -C build-dir=build/py -C editable.rebuild=true +(cd "$(mktemp -d)" && uv run --project "$OLDPWD" \ + pytest -o python_files='*_Py.py' "$OLDPWD/src/axom/sidre/tests/") +``` + +Note that Axom's Python tests are named `*_Py.py`, which pytest's default `python_files` patterns do not match +and several tests write output files into the current directory, so we run them from a scratch directory. + +### Stable ABI (abi3) + +By default the wheel is tagged for the exact CPython that built it. +With CMake >= 3.26 and Python >= 3.12, opt into a single abi3 wheel that serves +every CPython >= 3.12 on the machine by passing both flags together +(the CMake option makes nanobind build the limited-API module; +the scikit-build-core setting sets the wheel tag, and the two must agree): + +```bash +uv build --wheel \ + -C cmake.define.AXOM_PYTHON_STABLE_ABI=ON \ + -C wheel.py-api=cp312 \ + -C cmake.define.AXOM_DIR="$AXOM_INSTALL/lib/cmake" \ + src/python +``` + +Below Python 3.12 nanobind silently builds a non-stable module, +so only enable this on a 3.12+ interpreter. CMake's `FindPython` needs its +`Development.SABIModule` component for this path, which is available starting in CMake 3.26. +The build fails with an explicit message if either prerequisite is missing, +rather than quietly producing a mislabelled wheel. +Stable ABI relaxes the Python-version coupling, not the toolchain coupling: +an abi3 wheel is still specific to the host-config it was built against. +Free-threaded (`abi3t`) wheels are not built today; scikit-build-core 1.0+ +can emit those tags once the bindings and Conduit run under a free-threaded interpreter. + +### Package metadata and extras -- These files are installed verbatim (no template substitution). They contain no CMake-configured values. -- A `pyproject.toml` for the standalone wheel is not present yet. We will add it in the future when we add the wheel. - Until then this directory is consumed only by the CMake build. -- End-user instructions for installing and importing the bindings currently live in the Sidre user guide's "Python interface" page. +Wheel metadata is static, but whether the underlying Axom is an MPI build is a build-time choice, +so the wheel cannot force MPI dependencies at install time. +The `mpi` extra declares `mpi4py`, and the `test` extra declares `pytest`. +Runtime dependencies intentionally stay minimal: `numpy` is required, +while Conduit's Python module is exposed by the generated `conduit.pth` file. +The GitHub wheel test lane builds against an explicitly passed prebuilt Axom +install, passes the matching host-config through `cmake.args=-C`, and selects +the `mpi` extra automatically when that host-config reports `ENABLE_MPI=ON`. diff --git a/src/python/cmake/axom-python-env.sh.in b/src/python/cmake/axom-python-env.sh.in new file mode 100644 index 0000000000..6b350782e8 --- /dev/null +++ b/src/python/cmake/axom-python-env.sh.in @@ -0,0 +1,73 @@ +# Copyright (c) Lawrence Livermore National Security, LLC and other +# Axom Project Contributors. See top-level LICENSE and COPYRIGHT +# files for dates and other details. +# +# SPDX-License-Identifier: (BSD-3-Clause) + +# Generated by Axom's Python wheel build. +# Source this bash file to expose the Axom install this wheel was built against: +# . /axom/share/axom-python-env.sh +# +# Two groups of variables are set: +# +# * Variables honored by a subsequent `cmake` invocation. CMake's find_package() reads +# _DIR from the environment, and its compiler detection reads CC and CXX. +# +# * Informational variables that capture the build environment +# +# To reproduce Axom's full toolchain: +# +# cmake -C "$AXOM_PYTHON_HOST_CONFIG" -S -B +# +# Empty values are skipped rather than exported, so a setting this wheel has +# nothing to say about (e.g. the MPI wrappers of a non-MPI build) is left alone. + +_axom_python_env_script="${BASH_SOURCE[0]}" +_axom_python_share_dir=$(CDPATH= cd -- "$(dirname -- "${_axom_python_env_script}")" && pwd) +_axom_python_package_dir=$(CDPATH= cd -- "${_axom_python_share_dir}/.." && pwd) +_axom_python_site_packages_dir=$(CDPATH= cd -- "${_axom_python_package_dir}/.." && pwd) +_axom_python_version_dir=$(CDPATH= cd -- "${_axom_python_site_packages_dir}/.." && pwd) +_axom_python_lib_dir=$(CDPATH= cd -- "${_axom_python_version_dir}/.." && pwd) +_axom_python_prefix_dir=$(CDPATH= cd -- "${_axom_python_lib_dir}/.." && pwd) + +# export NAME=VALUE, skipping empty values +_axom_python_export() { + if [ -n "$2" ]; then + export "$1=$2" + fi +} + +# --- honored by cmake ------------------------------------------------------- +_axom_python_export AXOM_PYTHON_HOST_CONFIG "${_axom_python_share_dir}/axom-python-host-config.cmake" +_axom_python_export axom_DIR "@axom_DIR@" +_axom_python_export AXOM_DIR "@axom_DIR@" +_axom_python_export Conduit_DIR "@_axom_py_conduit_cmake_dir@" +_axom_python_export CC "@CMAKE_C_COMPILER@" +_axom_python_export CXX "@CMAKE_CXX_COMPILER@" + +# --- informational ---------------------------------------------------------- +_axom_python_export AXOM_INSTALL_PREFIX "@AXOM_INSTALL_PREFIX@" +_axom_python_export CONDUIT_DIR "@AXOM_CONDUIT_DIR@" +_axom_python_export CONDUIT_PYTHON_MODULE_DIR "@_axom_py_conduit_python_module_dir@" +_axom_python_export ENABLE_MPI "@AXOM_USE_MPI@" +_axom_python_export AXOM_ENABLE_MPI "@AXOM_USE_MPI@" +_axom_python_export CMAKE_C_COMPILER "@CMAKE_C_COMPILER@" +_axom_python_export CMAKE_CXX_COMPILER "@CMAKE_CXX_COMPILER@" +if [ "@AXOM_USE_MPI@" = "ON" ] || [ "@AXOM_USE_MPI@" = "TRUE" ] || [ "@AXOM_USE_MPI@" = "1" ]; then + _axom_python_export MPI_C_COMPILER "@MPI_C_COMPILER@" + _axom_python_export MPI_CXX_COMPILER "@MPI_CXX_COMPILER@" +fi +if [ -x "${_axom_python_prefix_dir}/bin/python" ]; then + _axom_python_export Python_EXECUTABLE "${_axom_python_prefix_dir}/bin/python" +else + _axom_python_export Python_EXECUTABLE "@Python_EXECUTABLE@" +fi + +unset -f _axom_python_export +unset _axom_python_share_dir +unset _axom_python_env_script +unset _axom_python_package_dir +unset _axom_python_site_packages_dir +unset _axom_python_version_dir +unset _axom_python_lib_dir +unset _axom_python_prefix_dir diff --git a/src/python/cmake/axom-python-host-config.cmake.in b/src/python/cmake/axom-python-host-config.cmake.in new file mode 100644 index 0000000000..eace669d3a --- /dev/null +++ b/src/python/cmake/axom-python-host-config.cmake.in @@ -0,0 +1,73 @@ +# Copyright (c) Lawrence Livermore National Security, LLC and other +# Axom Project Contributors. See top-level LICENSE and COPYRIGHT +# files for dates and other details. +# +# SPDX-License-Identifier: (BSD-3-Clause) + +# Generated by Axom's Python wheel build. +# Use with: +# cmake -C -S -B + +get_filename_component(_AXOM_PYTHON_SHARE_DIR "${CMAKE_CURRENT_LIST_FILE}" DIRECTORY) +get_filename_component(_AXOM_PYTHON_PACKAGE_DIR "${_AXOM_PYTHON_SHARE_DIR}" DIRECTORY) +get_filename_component(_AXOM_PYTHON_SITE_PACKAGES_DIR "${_AXOM_PYTHON_PACKAGE_DIR}" DIRECTORY) +get_filename_component(_AXOM_PYTHON_VERSION_DIR "${_AXOM_PYTHON_SITE_PACKAGES_DIR}" DIRECTORY) +get_filename_component(_AXOM_PYTHON_LIB_DIR "${_AXOM_PYTHON_VERSION_DIR}" DIRECTORY) +get_filename_component(_AXOM_PYTHON_PREFIX_DIR "${_AXOM_PYTHON_LIB_DIR}" DIRECTORY) + +set(_AXOM_PYTHON_INSTALLED_EXECUTABLE "") +if(EXISTS "${_AXOM_PYTHON_PREFIX_DIR}/bin/python") + set(_AXOM_PYTHON_INSTALLED_EXECUTABLE "${_AXOM_PYTHON_PREFIX_DIR}/bin/python") +endif() + +set(axom_DIR "@axom_DIR@" CACHE PATH "Axom CMake package directory") +set(AXOM_DIR "@axom_DIR@" CACHE PATH "Axom CMake package directory") +set(AXOM_INSTALL_PREFIX "@AXOM_INSTALL_PREFIX@" CACHE PATH "Axom install prefix") + +set(Conduit_DIR "@_axom_py_conduit_cmake_dir@" CACHE PATH "Conduit CMake package directory") +set(CONDUIT_DIR "@AXOM_CONDUIT_DIR@" CACHE PATH "Conduit install prefix") +# NOTE: after find_package(axom), ConduitConfig.cmake may set the normal +# CONDUIT_PYTHON_MODULE_DIR variable to a path relative to Conduit's install root, +# shadowing this cache entry. Use AXOM_CONDUIT_PYTHON_MODULE_DIR instead +# since axom-config.cmake records it as an absolute path. +set(CONDUIT_PYTHON_MODULE_DIR "@_axom_py_conduit_python_module_dir@" CACHE PATH "Conduit Python module directory") + +# Axom's headers, and the third-party headers they include, require this standard. +# BLT_CXX_STD is what BLT-based projects read; CMAKE_CXX_STANDARD covers the rest. +set(BLT_CXX_STD "c++@CMAKE_CXX_STANDARD@" CACHE STRING "C++ standard used for the Axom Python wheel") +set(CMAKE_CXX_STANDARD "@CMAKE_CXX_STANDARD@" CACHE STRING "C++ standard used for the Axom Python wheel") + +set(ENABLE_MPI "@AXOM_USE_MPI@" CACHE BOOL "Enable MPI to match the Axom Python wheel") +set(AXOM_ENABLE_MPI "@AXOM_USE_MPI@" CACHE BOOL "Whether this Axom Python wheel was built against MPI-enabled Axom") + +if(EXISTS "@CMAKE_C_COMPILER@") + set(CMAKE_C_COMPILER "@CMAKE_C_COMPILER@" CACHE FILEPATH "C compiler used for the Axom Python wheel") +endif() + +if(EXISTS "@CMAKE_CXX_COMPILER@") + set(CMAKE_CXX_COMPILER "@CMAKE_CXX_COMPILER@" CACHE FILEPATH "CXX compiler used for the Axom Python wheel") +endif() + +if(ENABLE_MPI) + if(EXISTS "@MPI_C_COMPILER@") + set(MPI_C_COMPILER "@MPI_C_COMPILER@" CACHE FILEPATH "MPI C compiler wrapper used for the Axom Python wheel") + endif() + + if(EXISTS "@MPI_CXX_COMPILER@") + set(MPI_CXX_COMPILER "@MPI_CXX_COMPILER@" CACHE FILEPATH "MPI CXX compiler wrapper used for the Axom Python wheel") + endif() +endif() + +if(_AXOM_PYTHON_INSTALLED_EXECUTABLE) + set(Python_EXECUTABLE "${_AXOM_PYTHON_INSTALLED_EXECUTABLE}" CACHE FILEPATH "Python interpreter for this Axom Python installation") +elseif(EXISTS "@Python_EXECUTABLE@") + set(Python_EXECUTABLE "@Python_EXECUTABLE@" CACHE FILEPATH "Python interpreter used for the Axom Python wheel") +endif() + +unset(_AXOM_PYTHON_SHARE_DIR) +unset(_AXOM_PYTHON_PACKAGE_DIR) +unset(_AXOM_PYTHON_SITE_PACKAGES_DIR) +unset(_AXOM_PYTHON_VERSION_DIR) +unset(_AXOM_PYTHON_LIB_DIR) +unset(_AXOM_PYTHON_PREFIX_DIR) +unset(_AXOM_PYTHON_INSTALLED_EXECUTABLE) diff --git a/src/python/cmake/conduit.pth.in b/src/python/cmake/conduit.pth.in new file mode 100644 index 0000000000..735ea6f459 --- /dev/null +++ b/src/python/cmake/conduit.pth.in @@ -0,0 +1 @@ +@_axom_py_conduit_python_module_dir@ diff --git a/src/python/pyproject.toml b/src/python/pyproject.toml new file mode 100644 index 0000000000..fd393d83f1 --- /dev/null +++ b/src/python/pyproject.toml @@ -0,0 +1,116 @@ +# Copyright (c) Lawrence Livermore National Security, LLC and other +# Axom Project Contributors. See top-level LICENSE and COPYRIGHT +# files for dates and other details. +# +# SPDX-License-Identifier: (BSD-3-Clause) + +# Thin, binding-only wheel for Axom's Python package. +# +# This project compiles Axom's nanobind bindings against an already-installed Axom. +# It does not build Axom or any of its third-party libraries. +# Point it at an install with axom_DIR (NOT CMAKE_PREFIX_PATH, which scikit-build-core +# reserves for its own build environment -- see src/python/README.md): +# +# pip install ./src/python -C cmake.define.axom_DIR=$AXOM_INSTALL/lib/cmake +# +# The pure-Python package tree it wraps lives beside this file under src/ +# and has the same tree layout as the CMake-based installation (see src/python/README.md). + +[build-system] +# nanobind and scikit-build-core are build-time only. +# numpy is listed here too (in addition to [project.dependencies]) because the stub step +# imports the freshly-built module to introspect it, +# and nanobind's ndarray annotations need numpy importable at build time. +# +# conduit is deliberately not required at build time: +# import_conduit() runs lazily inside the Node conversion helpers, +# not at module import, so `import _sidre` does not touch it. +requires = ["scikit-build-core>=1.0", "nanobind>=2.7.0", "numpy>=1.22"] +build-backend = "scikit_build_core.build" + +[project] +name = "axom" +# Version is sourced dynamically from the C++ library's canonical version file (src/cmake/AxomVersion.cmake) +# at build time via [[tool.dynamic-metadata]] below, so the wheel version cannot drift from libaxom. +# axom.__init__ reads it back with importlib.metadata.version("axom") so the installed __version__ resolves. +dynamic = ["version"] +description = "Python bindings for LLNL Axom, a CS infrastructure library for HPC applications" +readme = "README.md" +requires-python = ">=3.10" +license = "BSD-3-Clause" +authors = [{ name = "Axom Project Contributors" }] +keywords = ["axom", "hpc"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "Programming Language :: C++", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering", +] +# conduit is a hard runtime dependency for sidre that intentionally does NOT appear here: +# the PyPI name 'conduit' is an unrelated project, and 'llnl-conduit' is a separate build of +# Conduit that is unlikely to be ABI-compatible with the one Axom links +# -- either way pip would put a second libconduit in the process (see src/python/README.md). +# The conduit Python module must come from the same Conduit build Axom links, exposed via a .pth file. +# Revisit if Conduit ever ships a thin, find_package-based binding wheel. +dependencies = ["numpy>=1.22"] + +[project.optional-dependencies] +# Wheel metadata is static, but MPI-ness is a build configuration: +# a wheel built from a +mpi Axom cannot force this extra at install time. +# The README and the import-error guidance tell users to `pip install 'axom[mpi]'` when needed. +mpi = ["mpi4py>=3.1"] +test = ["pytest"] + +[project.urls] +Homepage = "https://github.com/LLNL/axom" +Documentation = "https://axom.readthedocs.io" +Source = "https://github.com/LLNL/axom" + +[project.scripts] +axom-python-config = "axom.config:main" + +# Source the wheel version from the C++ library's canonical version file, so it can never drift from libaxom. +# Build from a full repo checkout: the sdist does not carry this out-of-tree file, +# and standalone-sdist/PyPI is out of scope (see the sdist note under [tool.scikit-build]). +[[tool.dynamic-metadata]] +provider = "scikit_build_core.metadata.regex" +field = "version" +input = "../cmake/AxomVersion.cmake" +regex = '''(?sx) +set\( AXOM_VERSION_MAJOR \s+ (?P\d+) \) .*? +set\( AXOM_VERSION_MINOR \s+ (?P\d+) \) .*? +set\( AXOM_VERSION_PATCH \s+ (?P\d+) \) +''' +result = "{major}.{minor}.{patch}" + +[tool.scikit-build] +minimum-version = "build-system.requires" +build-dir = "build/{wheel_tag}" +# The 'axom' regular package (its __init__.py files, py.typed and the checked-in package stub) ships as pure Python from src/axom. +# The compiled extension and its generated _sidre.pyi are installed by CMakeLists.txt into axom/sidre/. +wheel.packages = ["src/axom"] + +# NOTE on the sdist: the binding translation unit lives with its component (../axom/sidre/nanobind_sidre.cpp), outside this project directory. +# A `sdist.include = ["../axom/..."]` entry does NOT vendor it -- scikit-build-core restricts sdist contents to the project root +# and silently drops out-of-tree paths (verified with scikit-build-core 1.0.3). +# The resulting sdist is therefore not self-contained: the supported build paths +# compile from a full repo checkout (`pip install ./src/python`, `uv build src/python`), +# where CMakeLists.txt reads the TU from its on-disk relative path. +# Standalone-sdist / PyPI distribution is out of scope. +# If that ever changes, vendor the TU into this tree during a pre-sdist step (or move the project root above the TU). +# +# STABLE_ABI / abi3 (cp312): opt-in, so the default build produces per-Python-version wheel tags. +# To build one abi3 wheel that serves every CPython >= 3.12 on the machine, use CMake >= 3.26 +# and Python >= 3.12, and pass BOTH of these at build time. They must agree, since the first +# makes nanobind build the limited-API module and the second sets the wheel tag: +# uv build --wheel -C cmake.define.AXOM_PYTHON_STABLE_ABI=ON -C wheel.py-api=cp312 ... +# +# wheel.py-api is intentionally left unset here rather than hard-coded to cp312: +# hard-coding it would tag every wheel cp312 even when built non-stable on < 3.12. +# wheel.py-api = "cp312" # set via -C wheel.py-api=cp312 in the abi3 build only + + +[tool.scikit-build.cmake.define] +# Resolve libsidre/libconduit/HDF5 from their install locations at runtime with no LD_LIBRARY_PATH. +CMAKE_INSTALL_RPATH_USE_LINK_PATH = "ON" diff --git a/src/python/src/axom/config.py b/src/python/src/axom/config.py new file mode 100644 index 0000000000..588e58a047 --- /dev/null +++ b/src/python/src/axom/config.py @@ -0,0 +1,92 @@ +# Copyright (c) Lawrence Livermore National Security, LLC and other +# Axom Project Contributors. See top-level LICENSE and COPYRIGHT +# files for dates and other details. +# +# SPDX-License-Identifier: (BSD-3-Clause) + +"""Helpers for locating Axom Python wheel configuration files. + +The files these helpers point at (``axom-python-host-config.cmake`` and ``axom-python-env.sh``) +are generated by the pip/uv wheel build in ``src/python``. +Axom's in-tree CMake build installs the same ``axom`` package but does not generate them, +so on such an installation the accessors below raise :class:`FileNotFoundError` +rather than returning a path that does not exist. +Use :func:`has_wheel_config` to test without raising. +""" + +from __future__ import annotations + +import argparse +import sys +from importlib import resources +from pathlib import Path + +_MISSING_HINT = ( + "This is expected for an Axom installed by its CMake build: the host-config " + "and environment script are generated only by the pip/uv wheel build in " + "src/python. Reinstall with `pip install /src/python` to get them." +) + + +def share_dir() -> Path: + """Return the installed Axom Python package share directory. + + The directory is not required to exist; see :func:`has_wheel_config`. + """ + return Path(resources.files("axom").joinpath("share")) + + +def has_wheel_config() -> bool: + """Return True when the wheel-generated configuration files are present.""" + return (share_dir() / "axom-python-host-config.cmake").is_file() + + +def _require(path: Path, what: str) -> Path: + if not path.is_file(): + raise FileNotFoundError(f"{what} not found at {path}. {_MISSING_HINT}") + return path + + +def host_config_path() -> Path: + """Return the CMake host-config generated for this Axom Python wheel. + + :raises FileNotFoundError: if this installation carries no generated host-config. + """ + return _require(share_dir() / "axom-python-host-config.cmake", "Axom Python host-config") + + +def env_script_path() -> Path: + """Return the shell environment helper generated for this Axom Python wheel. + + :raises FileNotFoundError: if this installation carries no generated env script. + """ + return _require(share_dir() / "axom-python-env.sh", "Axom Python environment script") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Print Axom Python wheel configuration paths.") + group = parser.add_mutually_exclusive_group() + group.add_argument("--host-config", action="store_true", help="print the CMake host-config path") + group.add_argument("--env-script", action="store_true", help="print the shell environment script path") + group.add_argument("--share-dir", action="store_true", help="print the Axom Python share directory") + group.add_argument("--cmake-args", action="store_true", help="print CMake arguments using the host-config") + args = parser.parse_args(argv) + + try: + if args.env_script: + print(env_script_path()) + elif args.share_dir: + print(share_dir()) + elif args.cmake_args: + print(f"-C {host_config_path()}") + else: + print(host_config_path()) + except FileNotFoundError as err: + print(err, file=sys.stderr) + return 1 + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())