Adds axom module to python interface with axom.sidre

RELEASE-NOTES.md

@@ -49,6 +49,8 @@ The Axom project release numbers follow [Semantic Versioning](http://semver.org/
 - Slam: Adds `make_*_set`, `make_*_relation` and `make_map` helper functions for building sets, relations and maps
 - Primal: Adds `primal::Sphere::contains(const Point&, bool includeBoundary = true)` to efficiently test whether
   a point lies within a sphere. Use `getOrientation()` when a tolerance-aware boundary classification is needed.
+- Python: Adds the `AXOM_PYTHON_MODULE_INSTALL_PREFIX` CMake variable to control where Axom installs its Python
+  package(s), relative to the install prefix.
 
 ### Removed
 - Bump: Removed `axom::bump::views::MultiBufferMaterialView`, which was a view type for an obsolete flavor of Blueprint matset.
@@ -65,6 +67,8 @@ The Axom project release numbers follow [Semantic Versioning](http://semver.org/
 - Core: Optimization for axom::Array indirection -- since the stride is always 1, we can remove the runtime multiplication
 - Python: Removes build and test dependencies from `run_python_with_axom.sh` wrapper script
 - Changed to `#pragma once` instead of unique header guard defines
+- Python: Sidre's bindings now install under the `axom` Python package (`import axom.sidre`)
+  Code that previously imported `pysidre` needs to be updated to `axom.sidre`.
 
 ### Fixed
 - Primal: Fixes signs of `compute_moments` to match orientation convention in `primal::evaluate_area_integral`

scripts/github-actions/linux-build_and_test.sh

@@ -30,22 +30,22 @@ export BUILD_TYPE=${BUILD_TYPE:-Debug}
 
 if [[ "$DO_BUILD" == "yes" ]] ; then
     echo "~~~~~~ FIND NUMPROCS ~~~~~~~~"
-    NUMPROCS=`python3 -c "import os; print(f'{os.cpu_count()}')"`
-    NUM_BUILD_PROCS=`python3 -c "import os; print(f'{max(2, os.cpu_count() * 8 // 10)}')"`
+    NUMPROCS=$(python3 -c 'import os; print(os.cpu_count())')
+    NUM_BUILD_PROCS=$(python3 -c 'import os; print(max(2, os.cpu_count() * 8 // 10))')
 
     echo "~~~~~~ RUNNING CMAKE ~~~~~~~~"
     or_die python3 ./config-build.py -bp builddir -hc ./host-configs/docker/${HOST_CONFIG} -bt ${BUILD_TYPE} -DENABLE_GTEST_DEATH_TESTS=ON ${CMAKE_EXTRA_FLAGS}
     or_die cd builddir
 
     echo "~~~~~~ BUILDING ~~~~~~~~"
     if [[ ${CMAKE_EXTRA_FLAGS} == *COVERAGE* ]] ; then
-        or_die make -j $NUM_BUILD_PROCS
+        or_die make -j ${NUM_BUILD_PROCS}
     else
-        or_die make -j $NUM_BUILD_PROCS VERBOSE=1
+        or_die make -j ${NUM_BUILD_PROCS} VERBOSE=1
     fi
 
     echo "~~~~~~ RUNNING TESTS ~~~~~~~~"
-    make CTEST_OUTPUT_ON_FAILURE=1 test ARGS='-T Test --output-on-failure -j$NUM_BUILD_PROCS'
+    make CTEST_OUTPUT_ON_FAILURE=1 test ARGS="-T Test --output-on-failure -j${NUM_BUILD_PROCS}"
 
     if [[ "${DO_BENCHMARKS}" == "yes" ]] ; then
         echo "~~~~~~ RUNNING BENCHMARKS ~~~~~~~~"
@@ -56,5 +56,16 @@ if [[ "$DO_BUILD" == "yes" ]] ; then
         echo "~~~~~~ RUNNING MEMCHECK ~~~~~~~~"
         or_die ctest -T memcheck
     fi
-fi
 
+    echo "~~~~~~ INSTALLING ~~~~~~~~"
+    or_die make install
+
+    # For configs that generated Python bindings, check that we can run a Python script with Axom
+    echo "~~~~~~ RUNNING INSTALLED PYTHON EXAMPLE ~~~~~~~~"
+    INSTALL_PREFIX=$(awk -F= '/^CMAKE_INSTALL_PREFIX:PATH=/{print $2}' CMakeCache.txt)
+    PYTHON_RUNNER="${INSTALL_PREFIX}/bin/run_python_with_axom.sh"
+    PYTHON_EXAMPLE="${INSTALL_PREFIX}/examples/axom/using-with-python/example.py"
+    if [[ -x "${PYTHON_RUNNER}" && -f "${PYTHON_EXAMPLE}" ]] ; then
+        or_die "${PYTHON_RUNNER}" "${PYTHON_EXAMPLE}"
+    fi
+fi

scripts/spack/packages/axom/package.py

@@ -5,6 +5,7 @@
 import os
 import shutil
 import socket
+import tempfile
 from os.path import join as pjoin
 
 from spack_repo.builtin.build_systems.cached_cmake import (
@@ -277,15 +278,20 @@ class Axom(CachedCMakePackage, CudaPackage, ROCmPackage):
         depends_on("mfem~mpi", when="~mpi")
         depends_on("mfem@4.5.0:", when="@0.7.0:")
 
-    depends_on("python", when="+python")
-
     # Python
     with when("+python"):
+        depends_on("python")
+
+        # extending python allows spack environment views to import axom from python
+        extends("python")
+
         depends_on("py-nanobind@2.7.0:")
         depends_on("py-pytest")
+        depends_on("py-packaging")
+        depends_on("py-pygments")
         depends_on("py-numpy")
         depends_on("py-mpi4py", when="+mpi")
-        depends_on("conduit+python")
+        depends_on("conduit+python", when="+conduit")
 
     # Devtools
     with when("+devtools"):
@@ -757,6 +763,16 @@ def initconfig_package_entries(self):
             python_bin_dir = get_spec_path(spec, "python", path_replacements, use_bin=True)
             entries.append(cmake_cache_path("Python_EXECUTABLE", pjoin(python_bin_dir, "python3")))
 
+        if spec.satisfies("+python"):
+            # Install Axom's Python package(s) so a spack environment view merges them into
+            # a single site-packages and `import axom.sidre` works without updating PYTHONPATH
+            entries.append(
+                cmake_cache_path(
+                    "AXOM_PYTHON_MODULE_INSTALL_PREFIX",
+                    spec["python"].package.platlib,
+                )
+            )
+
         if spec.satisfies("^py-jsonschema"):
             jsonschema_dir = get_spec_path(spec, "py-jsonschema", path_replacements, use_bin=True)
             jsonschema_path = os.path.join(jsonschema_dir, "jsonschema")
@@ -787,20 +803,23 @@ def initconfig_package_entries(self):
                 )
 
         if spec.satisfies("+python"):
+            python_platlib = spec["python"].package.platlib
+
             # pytest requires pluggy and iniconfig
+            # newer pytest releases also import packaging/pygments from separate Spack prefixes.
             for dep in (
                 "py-nanobind",
                 "py-pytest",
                 "py-numpy",
                 "py-pluggy",
                 "py-iniconfig",
+                "py-packaging",
+                "py-pygments",
                 "py-mpi4py",
             ):
                 if spec.satisfies("^{0}".format(dep)):
-                    dep_dir = get_spec_path(spec, dep, path_replacements, use_lib=True)
-                    py_libdir = join_path(
-                        dep_dir, f"python{spec['python'].version.up_to(2)}", "site-packages"
-                    )
+                    dep_dir = get_spec_path(spec, dep, path_replacements)
+                    py_libdir = join_path(dep_dir, python_platlib)
                     entries.append(
                         cmake_cache_path("%s_DIR" % dep.upper().replace("-", "_"), py_libdir)
                     )
@@ -845,7 +864,8 @@ def build_test(self):
     def test_install_using_cmake(self):
         """build example with cmake and run"""
         example_src_dir = join_path(self.prefix.examples.axom, "using-with-cmake")
-        example_stage_dir = "./cmake"
+        example_test_dir = tempfile.mkdtemp(prefix="axom-cmake-example-")
+        example_stage_dir = join_path(example_test_dir, "using-with-cmake")
         shutil.copytree(example_src_dir, example_stage_dir)
         with working_dir(join_path(example_stage_dir, "build"), create=True):
             cmake_args = ["-C ../host-config.cmake", example_src_dir]
@@ -861,10 +881,60 @@ def test_install_using_cmake(self):
     def test_install_using_make(self):
         """build example with make and run"""
         example_src_dir = join_path(self.prefix.examples.axom, "using-with-make")
-        example_stage_dir = "./make"
+        example_test_dir = tempfile.mkdtemp(prefix="axom-make-example-")
+        example_stage_dir = join_path(example_test_dir, "using-with-make")
         shutil.copytree(example_src_dir, example_stage_dir)
         with working_dir(example_stage_dir, create=True):
             make(f"AXOM_DIR={self.prefix}")
             example = Executable("./example")
             example()
             make("clean")
+
+    @run_after("install", when="+examples+python+tools components=sidre")
+    @on_package_attributes(run_tests=True)
+    def test_install_using_python(self):
+        """run python example against installed axom"""
+        example = join_path(self.prefix.examples.axom, "using-with-python", "example.py")
+        python_runner = join_path(self.prefix.bin, "run_python_with_axom.sh")
+        if not os.path.isfile(example):
+            raise RuntimeError("Missing installed python example: {0}".format(example))
+        if not os.path.isfile(python_runner):
+            raise RuntimeError("Missing installed python runner: {0}".format(python_runner))
+        run_python = Executable(python_runner)
+        run_python(example)
+
+    @run_after("install", when="+python components=sidre")
+    @on_package_attributes(run_tests=True)
+    def test_axom_sidre_installed_into_site_packages(self):
+        """Check axom.sidre installed into a site-packages-shaped prefix
+        and imports from view-shaped site-packages paths.
+        """
+        python_pkg = self.spec["python"].package
+        python_platlib = python_pkg.platlib
+        site_packages = join_path(self.prefix, python_platlib)
+        sidre_pkg_dir = join_path(site_packages, "axom", "sidre")
+        if not os.path.isdir(sidre_pkg_dir):
+            raise RuntimeError(
+                "axom.sidre was not installed under the interpreter platlib: "
+                "{0}".format(sidre_pkg_dir)
+            )
+
+        # Assemble the Python package directories a view would merge into site-packages.
+        import_path = [site_packages]
+        if self.spec.satisfies("+conduit"):
+            for conduit_py in (
+                join_path(self.spec["conduit"].prefix, python_platlib),
+                join_path(self.spec["conduit"].prefix, "python-modules"),
+            ):
+                if os.path.isdir(conduit_py):
+                    import_path.append(conduit_py)
+
+        for dep in ("py-numpy", "py-mpi4py"):
+            if self.spec.satisfies("^{0}".format(dep)):
+                dep_py = join_path(self.spec[dep].prefix, python_platlib)
+                if os.path.isdir(dep_py):
+                    import_path.append(dep_py)
+
+        imports = "import axom.sidre as s; import numpy; print('axom.sidre', s.__version__)"
+        python = Executable(join_path(self.spec["python"].prefix.bin, "python3"))
+        python("-c", imports, extra_env={"PYTHONPATH": ":".join(import_path)})

src/axom/sidre/CMakeLists.txt

@@ -138,49 +138,118 @@ endif()
 
 
 if(NANOBIND_FOUND)
-    nanobind_add_module(pysidre nanobind_sidre.cpp)
+    # Python bindings for Sidre.
+    # The pure-Python package scaffolding lives once under src/python/src/
+
+    # site-packages-shaped install directory for Axom's Python package(s).
+    # Keep this relative to the install prefix so `cmake --install --prefix`
+    # relocates the Python package along with Axom's other install artifacts.
+    set(_axom_python_install_default
+        "lib/python${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}/site-packages")
+    set(_axom_python_install_description
+        "Install destination for Axom's Python package(s), relative to the install prefix")
+    set(AXOM_PYTHON_MODULE_INSTALL_PREFIX
+        "${_axom_python_install_default}"
+        CACHE PATH
+        "${_axom_python_install_description}")
+
+    if(IS_ABSOLUTE "${AXOM_PYTHON_MODULE_INSTALL_PREFIX}")
+        file(RELATIVE_PATH _axom_python_install_relpath
+             "${CMAKE_INSTALL_PREFIX}"
+             "${AXOM_PYTHON_MODULE_INSTALL_PREFIX}")
+        if(NOT _axom_python_install_relpath MATCHES "^\\.\\.")
+            set(AXOM_PYTHON_MODULE_INSTALL_PREFIX
+                "${_axom_python_install_relpath}"
+                CACHE PATH
+                "${_axom_python_install_description}"
+                FORCE)
+        endif()
+        unset(_axom_python_install_relpath)
+    endif()
+    unset(_axom_python_install_default)
+    unset(_axom_python_install_description)
+
+    # Root of the staged package tree in the build directory.
+    # The build-tree interpreter (and run_python_with_axom.sh, via _PYEXT_DIR)
+    # puts this single directory on PYTHONPATH and then 'import axom.sidre' works.
+    set(_axom_py_build_root "${PROJECT_BINARY_DIR}/python")
+    set(_axom_py_pkg_src    "${CMAKE_CURRENT_SOURCE_DIR}/../../python/src")
+
+    # Build the extension under the shared 'axom' nanobind domain. 
+    # All of Axom's extension modules (currently just axom.sidre) share NB_DOMAIN=axom
+    # so that C++ types bound in one module, e.g. a sidre::Group*,
+    # recognized when passed to another module built from the same Axom build.
+    # nanobind only shares type bindings across modules that agree on 
+    # domain *and* nanobind ABI, compiler, and build mode.
+    nanobind_add_module(_sidre nanobind_sidre.cpp NB_DOMAIN axom)
     # conduit::conduit_python provides conduit_python.hpp
     # and is needed only by the binding translation unit, not by libsidre
-    target_link_libraries(pysidre PRIVATE sidre conduit::conduit_python)
+    target_link_libraries(_sidre PRIVATE sidre conduit::conduit_python)
+
+    # Place the built extension directly into the staged package tree so the
+    # build tree is import-ready without an extra copy step.
+    set_target_properties(_sidre PROPERTIES
+        LIBRARY_OUTPUT_DIRECTORY "${_axom_py_build_root}/axom/sidre")
 
     # Use HIP executable linker flags for the python module
     # (CMake treats modules separately from executables,
     # executable flags not automatically applied)
     if(AXOM_ENABLE_HIP)
-        # Make CMake compile the pysidre file with the HIP compiler.
+        # Make CMake compile the binding file with the HIP compiler.
         set_source_files_properties(nanobind_sidre.cpp PROPERTIES LANGUAGE HIP)
 
         string (REPLACE " " ";" MODULE_LINK_FLAGS "${CMAKE_EXE_LINKER_FLAGS}")
-        target_link_options(pysidre PRIVATE ${MODULE_LINK_FLAGS})
+        target_link_options(_sidre PRIVATE ${MODULE_LINK_FLAGS})
     endif()
 
-    install(TARGETS pysidre LIBRARY DESTINATION lib)
+    # Stage the pure-Python package scaffolding into the build tree at configure
+    # time (axom/ namespace root + py.typed, axom/sidre/ re-export).
+    axom_configure_file("${_axom_py_pkg_src}/axom/__init__.py"
+                        "${_axom_py_build_root}/axom/__init__.py" COPYONLY)
+    axom_configure_file("${_axom_py_pkg_src}/axom/py.typed"
+                        "${_axom_py_build_root}/axom/py.typed" COPYONLY)
+    axom_configure_file("${_axom_py_pkg_src}/axom/sidre/__init__.py"
+                        "${_axom_py_build_root}/axom/sidre/__init__.py" COPYONLY)
+    # Hand-written package stub: re-exports the generated _sidre.pyi statically
+    # so type checkers can see axom.sidre's surface
+    axom_configure_file("${_axom_py_pkg_src}/axom/sidre/__init__.pyi"
+                        "${_axom_py_build_root}/axom/sidre/__init__.pyi" COPYONLY)
 
-    # Type stubs (PEP 561). nanobind_add_stub imports the module to introspect it,
-    # so its runtime dependencies (conduit for Node interop, numpy for ndarray returns)
-    # must be importable during the build. We seed PYTHON_PATH with the module's 
-    # output directory plus the conduit/numpy install dirs from their cache variables when set;
-    # on an interpreter that already has conduit and numpy on its path these extra entries are harmless.
-    set(_pysidre_stub_pythonpath $<TARGET_FILE_DIR:pysidre>)
+    # Type stubs (PEP 561). nanobind_add_stub imports the module by its bare name
+    # ('import _sidre'), so the directory holding the built extension must be on
+    # PYTHON_PATH, along with the module's runtime deps (conduit for Node interop,
+    # numpy for ndarray returns). On an interpreter that already has conduit/numpy
+    # these extra entries are harmless.
+    set(_sidre_stub_pythonpath "${_axom_py_build_root}/axom/sidre")
     if(CONDUIT_PYTHON_MODULE_DIR)
-        list(APPEND _pysidre_stub_pythonpath ${CONDUIT_PYTHON_MODULE_DIR})
+        list(APPEND _sidre_stub_pythonpath ${CONDUIT_PYTHON_MODULE_DIR})
     endif()
     if(PY_NUMPY_DIR)
-        list(APPEND _pysidre_stub_pythonpath ${PY_NUMPY_DIR})
+        list(APPEND _sidre_stub_pythonpath ${PY_NUMPY_DIR})
     endif()
 
     nanobind_add_stub(
-        pysidre_stub
-        MODULE pysidre
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/pysidre.pyi"
-        MARKER_FILE "${CMAKE_CURRENT_BINARY_DIR}/py.typed"
-        PYTHON_PATH ${_pysidre_stub_pythonpath}
-        DEPENDS pysidre)
-
-    # Install the stub and py.typed marker next to the extension module so type checkers (mypy, pyright) can find them
-    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pysidre.pyi"
-                  "${CMAKE_CURRENT_BINARY_DIR}/py.typed"
-            DESTINATION lib)
+        _sidre_stub
+        MODULE _sidre
+        OUTPUT "${_axom_py_build_root}/axom/sidre/_sidre.pyi"
+        PYTHON_PATH ${_sidre_stub_pythonpath}
+        DEPENDS _sidre)
+
+    #--------------------------------------------------------------------------
+    # Install the package tree into the site-packages-shaped prefix.
+    #--------------------------------------------------------------------------
+    install(TARGETS _sidre
+            LIBRARY DESTINATION "${AXOM_PYTHON_MODULE_INSTALL_PREFIX}/axom/sidre")
+
+    install(FILES "${_axom_py_build_root}/axom/sidre/_sidre.pyi"
+                  "${_axom_py_pkg_src}/axom/sidre/__init__.py"
+                  "${_axom_py_pkg_src}/axom/sidre/__init__.pyi"
+            DESTINATION "${AXOM_PYTHON_MODULE_INSTALL_PREFIX}/axom/sidre")
+
+    # Namespace-root package files install once (not per component).
+    install(FILES "${_axom_py_pkg_src}/axom/__init__.py"
+                  "${_axom_py_pkg_src}/axom/py.typed"
+            DESTINATION "${AXOM_PYTHON_MODULE_INSTALL_PREFIX}/axom")
 endif()
 
 

src/axom/sidre/docs/sphinx/index.rst

@@ -99,3 +99,4 @@ needs and use cases.
    parallel_io_concepts
    sidre_conduit
    mfem_sidre_datacollection
+   python_interface