diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 97229006f9..86d22a0314 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -49,6 +49,8 @@ The Axom project release numbers follow [Semantic Versioning](http://semver.org/ - Slam: Adds `make_*_set`, `make_*_relation` and `make_map` helper functions for building sets, relations and maps - Primal: Adds `primal::Sphere::contains(const Point&, bool includeBoundary = true)` to efficiently test whether a point lies within a sphere. Use `getOrientation()` when a tolerance-aware boundary classification is needed. +- Python: Adds the `AXOM_PYTHON_MODULE_INSTALL_PREFIX` CMake variable to control where Axom installs its Python + package(s), relative to the install prefix. ### Removed - Bump: Removed `axom::bump::views::MultiBufferMaterialView`, which was a view type for an obsolete flavor of Blueprint matset. @@ -65,6 +67,8 @@ The Axom project release numbers follow [Semantic Versioning](http://semver.org/ - Core: Optimization for axom::Array indirection -- since the stride is always 1, we can remove the runtime multiplication - Python: Removes build and test dependencies from `run_python_with_axom.sh` wrapper script - Changed to `#pragma once` instead of unique header guard defines +- Python: Sidre's bindings now install under the `axom` Python package (`import axom.sidre`) + Code that previously imported `pysidre` needs to be updated to `axom.sidre`. ### Fixed - Primal: Fixes signs of `compute_moments` to match orientation convention in `primal::evaluate_area_integral` diff --git a/scripts/github-actions/linux-build_and_test.sh b/scripts/github-actions/linux-build_and_test.sh index a7a780ffe6..fc7ad631f5 100755 --- a/scripts/github-actions/linux-build_and_test.sh +++ b/scripts/github-actions/linux-build_and_test.sh @@ -30,8 +30,8 @@ export BUILD_TYPE=${BUILD_TYPE:-Debug} if [[ "$DO_BUILD" == "yes" ]] ; then echo "~~~~~~ FIND NUMPROCS ~~~~~~~~" - NUMPROCS=`python3 -c "import os; print(f'{os.cpu_count()}')"` - NUM_BUILD_PROCS=`python3 -c "import os; print(f'{max(2, os.cpu_count() * 8 // 10)}')"` + NUMPROCS=$(python3 -c 'import os; print(os.cpu_count())') + NUM_BUILD_PROCS=$(python3 -c 'import os; print(max(2, os.cpu_count() * 8 // 10))') echo "~~~~~~ RUNNING CMAKE ~~~~~~~~" or_die python3 ./config-build.py -bp builddir -hc ./host-configs/docker/${HOST_CONFIG} -bt ${BUILD_TYPE} -DENABLE_GTEST_DEATH_TESTS=ON ${CMAKE_EXTRA_FLAGS} @@ -39,13 +39,13 @@ if [[ "$DO_BUILD" == "yes" ]] ; then echo "~~~~~~ BUILDING ~~~~~~~~" if [[ ${CMAKE_EXTRA_FLAGS} == *COVERAGE* ]] ; then - or_die make -j $NUM_BUILD_PROCS + or_die make -j ${NUM_BUILD_PROCS} else - or_die make -j $NUM_BUILD_PROCS VERBOSE=1 + or_die make -j ${NUM_BUILD_PROCS} VERBOSE=1 fi echo "~~~~~~ RUNNING TESTS ~~~~~~~~" - make CTEST_OUTPUT_ON_FAILURE=1 test ARGS='-T Test --output-on-failure -j$NUM_BUILD_PROCS' + make CTEST_OUTPUT_ON_FAILURE=1 test ARGS="-T Test --output-on-failure -j${NUM_BUILD_PROCS}" if [[ "${DO_BENCHMARKS}" == "yes" ]] ; then echo "~~~~~~ RUNNING BENCHMARKS ~~~~~~~~" @@ -56,5 +56,16 @@ if [[ "$DO_BUILD" == "yes" ]] ; then echo "~~~~~~ RUNNING MEMCHECK ~~~~~~~~" or_die ctest -T memcheck fi -fi + echo "~~~~~~ INSTALLING ~~~~~~~~" + or_die make install + + # For configs that generated Python bindings, check that we can run a Python script with Axom + echo "~~~~~~ RUNNING INSTALLED PYTHON EXAMPLE ~~~~~~~~" + INSTALL_PREFIX=$(awk -F= '/^CMAKE_INSTALL_PREFIX:PATH=/{print $2}' CMakeCache.txt) + PYTHON_RUNNER="${INSTALL_PREFIX}/bin/run_python_with_axom.sh" + PYTHON_EXAMPLE="${INSTALL_PREFIX}/examples/axom/using-with-python/example.py" + if [[ -x "${PYTHON_RUNNER}" && -f "${PYTHON_EXAMPLE}" ]] ; then + or_die "${PYTHON_RUNNER}" "${PYTHON_EXAMPLE}" + fi +fi diff --git a/scripts/spack/packages/axom/package.py b/scripts/spack/packages/axom/package.py index a0f54db43c..cbefed5647 100644 --- a/scripts/spack/packages/axom/package.py +++ b/scripts/spack/packages/axom/package.py @@ -5,6 +5,7 @@ import os import shutil import socket +import tempfile from os.path import join as pjoin from spack_repo.builtin.build_systems.cached_cmake import ( @@ -277,15 +278,20 @@ class Axom(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on("mfem~mpi", when="~mpi") depends_on("mfem@4.5.0:", when="@0.7.0:") - depends_on("python", when="+python") - # Python with when("+python"): + depends_on("python") + + # extending python allows spack environment views to import axom from python + extends("python") + depends_on("py-nanobind@2.7.0:") depends_on("py-pytest") + depends_on("py-packaging") + depends_on("py-pygments") depends_on("py-numpy") depends_on("py-mpi4py", when="+mpi") - depends_on("conduit+python") + depends_on("conduit+python", when="+conduit") # Devtools with when("+devtools"): @@ -757,6 +763,16 @@ def initconfig_package_entries(self): python_bin_dir = get_spec_path(spec, "python", path_replacements, use_bin=True) entries.append(cmake_cache_path("Python_EXECUTABLE", pjoin(python_bin_dir, "python3"))) + if spec.satisfies("+python"): + # Install Axom's Python package(s) so a spack environment view merges them into + # a single site-packages and `import axom.sidre` works without updating PYTHONPATH + entries.append( + cmake_cache_path( + "AXOM_PYTHON_MODULE_INSTALL_PREFIX", + spec["python"].package.platlib, + ) + ) + if spec.satisfies("^py-jsonschema"): jsonschema_dir = get_spec_path(spec, "py-jsonschema", path_replacements, use_bin=True) jsonschema_path = os.path.join(jsonschema_dir, "jsonschema") @@ -787,20 +803,23 @@ def initconfig_package_entries(self): ) if spec.satisfies("+python"): + python_platlib = spec["python"].package.platlib + # pytest requires pluggy and iniconfig + # newer pytest releases also import packaging/pygments from separate Spack prefixes. for dep in ( "py-nanobind", "py-pytest", "py-numpy", "py-pluggy", "py-iniconfig", + "py-packaging", + "py-pygments", "py-mpi4py", ): if spec.satisfies("^{0}".format(dep)): - dep_dir = get_spec_path(spec, dep, path_replacements, use_lib=True) - py_libdir = join_path( - dep_dir, f"python{spec['python'].version.up_to(2)}", "site-packages" - ) + dep_dir = get_spec_path(spec, dep, path_replacements) + py_libdir = join_path(dep_dir, python_platlib) entries.append( cmake_cache_path("%s_DIR" % dep.upper().replace("-", "_"), py_libdir) ) @@ -845,7 +864,8 @@ def build_test(self): def test_install_using_cmake(self): """build example with cmake and run""" example_src_dir = join_path(self.prefix.examples.axom, "using-with-cmake") - example_stage_dir = "./cmake" + example_test_dir = tempfile.mkdtemp(prefix="axom-cmake-example-") + example_stage_dir = join_path(example_test_dir, "using-with-cmake") shutil.copytree(example_src_dir, example_stage_dir) with working_dir(join_path(example_stage_dir, "build"), create=True): cmake_args = ["-C ../host-config.cmake", example_src_dir] @@ -861,10 +881,60 @@ def test_install_using_cmake(self): def test_install_using_make(self): """build example with make and run""" example_src_dir = join_path(self.prefix.examples.axom, "using-with-make") - example_stage_dir = "./make" + example_test_dir = tempfile.mkdtemp(prefix="axom-make-example-") + example_stage_dir = join_path(example_test_dir, "using-with-make") shutil.copytree(example_src_dir, example_stage_dir) with working_dir(example_stage_dir, create=True): make(f"AXOM_DIR={self.prefix}") example = Executable("./example") example() make("clean") + + @run_after("install", when="+examples+python+tools components=sidre") + @on_package_attributes(run_tests=True) + def test_install_using_python(self): + """run python example against installed axom""" + example = join_path(self.prefix.examples.axom, "using-with-python", "example.py") + python_runner = join_path(self.prefix.bin, "run_python_with_axom.sh") + if not os.path.isfile(example): + raise RuntimeError("Missing installed python example: {0}".format(example)) + if not os.path.isfile(python_runner): + raise RuntimeError("Missing installed python runner: {0}".format(python_runner)) + run_python = Executable(python_runner) + run_python(example) + + @run_after("install", when="+python components=sidre") + @on_package_attributes(run_tests=True) + def test_axom_sidre_installed_into_site_packages(self): + """Check axom.sidre installed into a site-packages-shaped prefix + and imports from view-shaped site-packages paths. + """ + python_pkg = self.spec["python"].package + python_platlib = python_pkg.platlib + site_packages = join_path(self.prefix, python_platlib) + sidre_pkg_dir = join_path(site_packages, "axom", "sidre") + if not os.path.isdir(sidre_pkg_dir): + raise RuntimeError( + "axom.sidre was not installed under the interpreter platlib: " + "{0}".format(sidre_pkg_dir) + ) + + # Assemble the Python package directories a view would merge into site-packages. + import_path = [site_packages] + if self.spec.satisfies("+conduit"): + for conduit_py in ( + join_path(self.spec["conduit"].prefix, python_platlib), + join_path(self.spec["conduit"].prefix, "python-modules"), + ): + if os.path.isdir(conduit_py): + import_path.append(conduit_py) + + for dep in ("py-numpy", "py-mpi4py"): + if self.spec.satisfies("^{0}".format(dep)): + dep_py = join_path(self.spec[dep].prefix, python_platlib) + if os.path.isdir(dep_py): + import_path.append(dep_py) + + imports = "import axom.sidre as s; import numpy; print('axom.sidre', s.__version__)" + python = Executable(join_path(self.spec["python"].prefix.bin, "python3")) + python("-c", imports, extra_env={"PYTHONPATH": ":".join(import_path)}) diff --git a/src/axom/sidre/CMakeLists.txt b/src/axom/sidre/CMakeLists.txt index 750473d630..2091cb2fdd 100644 --- a/src/axom/sidre/CMakeLists.txt +++ b/src/axom/sidre/CMakeLists.txt @@ -138,49 +138,118 @@ endif() if(NANOBIND_FOUND) - nanobind_add_module(pysidre nanobind_sidre.cpp) + # Python bindings for Sidre. + # The pure-Python package scaffolding lives once under src/python/src/ + + # site-packages-shaped install directory for Axom's Python package(s). + # Keep this relative to the install prefix so `cmake --install --prefix` + # relocates the Python package along with Axom's other install artifacts. + set(_axom_python_install_default + "lib/python${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}/site-packages") + set(_axom_python_install_description + "Install destination for Axom's Python package(s), relative to the install prefix") + set(AXOM_PYTHON_MODULE_INSTALL_PREFIX + "${_axom_python_install_default}" + CACHE PATH + "${_axom_python_install_description}") + + if(IS_ABSOLUTE "${AXOM_PYTHON_MODULE_INSTALL_PREFIX}") + file(RELATIVE_PATH _axom_python_install_relpath + "${CMAKE_INSTALL_PREFIX}" + "${AXOM_PYTHON_MODULE_INSTALL_PREFIX}") + if(NOT _axom_python_install_relpath MATCHES "^\\.\\.") + set(AXOM_PYTHON_MODULE_INSTALL_PREFIX + "${_axom_python_install_relpath}" + CACHE PATH + "${_axom_python_install_description}" + FORCE) + endif() + unset(_axom_python_install_relpath) + endif() + unset(_axom_python_install_default) + unset(_axom_python_install_description) + + # Root of the staged package tree in the build directory. + # The build-tree interpreter (and run_python_with_axom.sh, via _PYEXT_DIR) + # puts this single directory on PYTHONPATH and then 'import axom.sidre' works. + set(_axom_py_build_root "${PROJECT_BINARY_DIR}/python") + set(_axom_py_pkg_src "${CMAKE_CURRENT_SOURCE_DIR}/../../python/src") + + # Build the extension under the shared 'axom' nanobind domain. + # All of Axom's extension modules (currently just axom.sidre) share NB_DOMAIN=axom + # so that C++ types bound in one module, e.g. a sidre::Group*, + # recognized when passed to another module built from the same Axom build. + # nanobind only shares type bindings across modules that agree on + # domain *and* nanobind ABI, compiler, and build mode. + nanobind_add_module(_sidre nanobind_sidre.cpp NB_DOMAIN axom) # conduit::conduit_python provides conduit_python.hpp # and is needed only by the binding translation unit, not by libsidre - target_link_libraries(pysidre PRIVATE sidre conduit::conduit_python) + target_link_libraries(_sidre PRIVATE sidre conduit::conduit_python) + + # Place the built extension directly into the staged package tree so the + # build tree is import-ready without an extra copy step. + set_target_properties(_sidre PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${_axom_py_build_root}/axom/sidre") # Use HIP executable linker flags for the python module # (CMake treats modules separately from executables, # executable flags not automatically applied) if(AXOM_ENABLE_HIP) - # Make CMake compile the pysidre file with the HIP compiler. + # Make CMake compile the binding file with the HIP compiler. set_source_files_properties(nanobind_sidre.cpp PROPERTIES LANGUAGE HIP) string (REPLACE " " ";" MODULE_LINK_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") - target_link_options(pysidre PRIVATE ${MODULE_LINK_FLAGS}) + target_link_options(_sidre PRIVATE ${MODULE_LINK_FLAGS}) endif() - install(TARGETS pysidre LIBRARY DESTINATION lib) + # Stage the pure-Python package scaffolding into the build tree at configure + # time (axom/ namespace root + py.typed, axom/sidre/ re-export). + axom_configure_file("${_axom_py_pkg_src}/axom/__init__.py" + "${_axom_py_build_root}/axom/__init__.py" COPYONLY) + axom_configure_file("${_axom_py_pkg_src}/axom/py.typed" + "${_axom_py_build_root}/axom/py.typed" COPYONLY) + axom_configure_file("${_axom_py_pkg_src}/axom/sidre/__init__.py" + "${_axom_py_build_root}/axom/sidre/__init__.py" COPYONLY) + # Hand-written package stub: re-exports the generated _sidre.pyi statically + # so type checkers can see axom.sidre's surface + axom_configure_file("${_axom_py_pkg_src}/axom/sidre/__init__.pyi" + "${_axom_py_build_root}/axom/sidre/__init__.pyi" COPYONLY) - # Type stubs (PEP 561). nanobind_add_stub imports the module to introspect it, - # so its runtime dependencies (conduit for Node interop, numpy for ndarray returns) - # must be importable during the build. We seed PYTHON_PATH with the module's - # output directory plus the conduit/numpy install dirs from their cache variables when set; - # on an interpreter that already has conduit and numpy on its path these extra entries are harmless. - set(_pysidre_stub_pythonpath $) + # Type stubs (PEP 561). nanobind_add_stub imports the module by its bare name + # ('import _sidre'), so the directory holding the built extension must be on + # PYTHON_PATH, along with the module's runtime deps (conduit for Node interop, + # numpy for ndarray returns). On an interpreter that already has conduit/numpy + # these extra entries are harmless. + set(_sidre_stub_pythonpath "${_axom_py_build_root}/axom/sidre") if(CONDUIT_PYTHON_MODULE_DIR) - list(APPEND _pysidre_stub_pythonpath ${CONDUIT_PYTHON_MODULE_DIR}) + list(APPEND _sidre_stub_pythonpath ${CONDUIT_PYTHON_MODULE_DIR}) endif() if(PY_NUMPY_DIR) - list(APPEND _pysidre_stub_pythonpath ${PY_NUMPY_DIR}) + list(APPEND _sidre_stub_pythonpath ${PY_NUMPY_DIR}) endif() nanobind_add_stub( - pysidre_stub - MODULE pysidre - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/pysidre.pyi" - MARKER_FILE "${CMAKE_CURRENT_BINARY_DIR}/py.typed" - PYTHON_PATH ${_pysidre_stub_pythonpath} - DEPENDS pysidre) - - # Install the stub and py.typed marker next to the extension module so type checkers (mypy, pyright) can find them - install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pysidre.pyi" - "${CMAKE_CURRENT_BINARY_DIR}/py.typed" - DESTINATION lib) + _sidre_stub + MODULE _sidre + OUTPUT "${_axom_py_build_root}/axom/sidre/_sidre.pyi" + PYTHON_PATH ${_sidre_stub_pythonpath} + DEPENDS _sidre) + + #-------------------------------------------------------------------------- + # Install the package tree into the site-packages-shaped prefix. + #-------------------------------------------------------------------------- + install(TARGETS _sidre + LIBRARY DESTINATION "${AXOM_PYTHON_MODULE_INSTALL_PREFIX}/axom/sidre") + + install(FILES "${_axom_py_build_root}/axom/sidre/_sidre.pyi" + "${_axom_py_pkg_src}/axom/sidre/__init__.py" + "${_axom_py_pkg_src}/axom/sidre/__init__.pyi" + DESTINATION "${AXOM_PYTHON_MODULE_INSTALL_PREFIX}/axom/sidre") + + # Namespace-root package files install once (not per component). + install(FILES "${_axom_py_pkg_src}/axom/__init__.py" + "${_axom_py_pkg_src}/axom/py.typed" + DESTINATION "${AXOM_PYTHON_MODULE_INSTALL_PREFIX}/axom") endif() diff --git a/src/axom/sidre/docs/sphinx/index.rst b/src/axom/sidre/docs/sphinx/index.rst index 14c3cfe462..3cf64fd36f 100644 --- a/src/axom/sidre/docs/sphinx/index.rst +++ b/src/axom/sidre/docs/sphinx/index.rst @@ -99,3 +99,4 @@ needs and use cases. parallel_io_concepts sidre_conduit mfem_sidre_datacollection + python_interface diff --git a/src/axom/sidre/docs/sphinx/python_interface.rst b/src/axom/sidre/docs/sphinx/python_interface.rst new file mode 100644 index 0000000000..f0c91e283a --- /dev/null +++ b/src/axom/sidre/docs/sphinx/python_interface.rst @@ -0,0 +1,153 @@ +.. ## Copyright (c) Lawrence Livermore National Security, LLC and other +.. ## Axom Project Contributors. See top-level LICENSE and COPYRIGHT +.. ## files for dates and other details. +.. ## +.. ## SPDX-License-Identifier: (BSD-3-Clause) + +****************************************************** +Python interface +****************************************************** + +Sidre ships a Python interface, ``axom.sidre``, that mirrors much of the C++ API, +e.g. to create a ``DataStore``, navigate ``Group`` and ``View`` objects, allocate and describe data, +and exchange data with `Conduit `_ ``Node`` objects and NumPy arrays without copying. +The interface is a compiled extension generated with `nanobind `_, +which is built when Axom is configured with the Sidre component and Python bindings enabled. + +.. code-block:: python + + import axom.sidre as sidre + + ds = sidre.DataStore() + root = ds.getRoot() + + grp = root.createGroup("fields") + view = grp.createViewAndAllocate("density", sidre.TypeID.FLOAT64_ID, 10) + + # Zero-copy NumPy view onto the buffer Sidre owns + arr = view.getDataArray() + arr[:] = 1.0 + + print(ds.getRoot().getView("fields/density").getNumElements()) # 10 + +The module carries a ``__version__`` matching the Axom release, and exposes +feature flags (e.g., ``AXOM_USE_HDF5``, ``AXOM_ENABLE_MPI``) so Python code can +branch on how Axom was built. + +======================================= +Getting a working ``import axom.sidre`` +======================================= + +How to make the interface importable depends on whether you are using an +installed Axom package or a build tree from an Axom development environment. +The two workflows are intentionally different. + +Development build tree +---------------------- + +Axom's uberenv-generated TPL environments intentionally use ``view: false``. +Those environments are for configuring and building Axom from a worktree. +They do not make the build-tree package importable by a plain interpreter when activated. + +For development builds, use CTest or the generated ``run_python_with_axom.sh`` helper. +These paths set the build-tree ``PYTHONPATH`` entries needed for Axom's staged package +and its Python runtime dependencies. + +.. code-block:: bash + + $ cd build-axom + $ ctest -R sidre_smoke_Py --output-on-failure + $ ./bin/run_python_with_axom.sh -c "import axom.sidre as sidre; print(sidre.__version__)" + +Spack environment view +---------------------- + +Axom declares itself a Python extension (``extends("python")``), so a spack +environment view can expose the bindings in the view's ``site-packages`` alongside their dependencies. +This is useful for testing or using an installed Axom package with a plain interpreter, +but it is not the normal Axom development-build workflow. + +To use this, install Axom with the ``+python`` variant in a dedicated +environment whose ``spack.yaml`` enables a view: + +.. code-block:: yaml + + spack: + specs: + - axom+python + view: true + ... + +After ``spack install``, the environment's interpreter should have a working Axom Python installation: + +.. code-block:: bash + + $ spack env activate . + $ python -c "import axom.sidre, conduit, numpy; print(axom.sidre.__version__)" + + +pip / uv wheel (thin, external Axom) +------------------------------------ + +.. note:: + + The pip/uv-installable wheel is planned and not yet available. + This section is a placeholder for the workflow it will enable. + Until it lands, use the build-tree helper for development builds + or a dedicated Spack environment view for installed-package testing. + +The wheel will compile only the binding code against an already-installed Axom +(located via ``CMAKE_PREFIX_PATH``); it will not build Axom or its third-party +libraries. Because a pip-built Conduit would produce a second, ABI-incompatible +``libconduit`` in the same process, the wheel will rely on the Conduit Python +module from the same Axom/Conduit build, exposed via a ``.pth`` file rather +than a PyPI install. + +==================================== +Working with Conduit and NumPy +==================================== + +Arrays returned by ``View.getDataArray`` and ``Buffer.getDataArray`` are +zero-copy NumPy views onto memory Sidre owns. The array keeps the owning Sidre +object alive for as long as the array is reachable. + +.. warning:: + + One sharp edge remains, and the binding cannot defend against it: + reallocating a buffer (for example growing a view) can move the underlying storage, + leaving any previously obtained NumPy array pointing at freed memory. + Re-acquire arrays after any operation that may reallocate, + exactly as you would re-slice a NumPy array after resizing its base. + +The ``conduit`` Python module is a hard runtime dependency of the bindings and +must wrap the same Conduit build Axom links. It imports alongside ``axom.sidre``: + +.. code-block:: python + + import axom.sidre as sidre + from conduit import Node + + n = Node() + n["field"] = 100 + assert n["field"] == 100 + +For how Sidre's on-disk layout and its in-memory hierarchy relate to the +Conduit Blueprint data model, see :doc:`sidre_conduit`. + +================================================================== +Running standalone scripts: the ``run_python_with_axom.sh`` helper +================================================================== + +The methods above make ``import axom.sidre`` work in a plain interpreter. +If you are not in a spack environment view and just want to run a one-off +Python script that uses Axom's Python modules, the build generates a helper script, +``run_python_with_axom.sh``, that prepends directories for the required runtime dependencies +to ``PYTHONPATH`` and then runs the interpreter: + +.. code-block:: bash + + $ ./bin/run_python_with_axom.sh my_script.py + $ ./bin/run_python_with_axom.sh -c "import axom.sidre, conduit" + +The helper is a ``PYTHONPATH`` prepend and is bash-only, so it does not compose +with Jupyter kernels, IDE runners, or debuggers diff --git a/src/axom/sidre/examples/CMakeLists.txt b/src/axom/sidre/examples/CMakeLists.txt index 23cde33e30..bafc03236d 100644 --- a/src/axom/sidre/examples/CMakeLists.txt +++ b/src/axom/sidre/examples/CMakeLists.txt @@ -154,11 +154,11 @@ if(NANOBIND_FOUND) axom_configure_file ("${example_source}" "${EXAMPLE_OUTPUT_DIRECTORY}/${example_source}" COPYONLY) - # Use convenience script to run python examples + # Run python examples directly under the interpreter (no wrapper). if(AXOM_ENABLE_PYTHON_TESTS) - axom_add_test ( + axom_add_python_test( NAME ${exe_name} - COMMAND ${PROJECT_BINARY_DIR}/bin/run_python_with_axom.sh ${EXAMPLE_OUTPUT_DIRECTORY}/${example_source} + COMMAND ${Python_EXECUTABLE} ${EXAMPLE_OUTPUT_DIRECTORY}/${example_source} ) endif() endforeach() diff --git a/src/axom/sidre/examples/sidre_createdatastore_Py.py b/src/axom/sidre/examples/sidre_createdatastore_Py.py index 0e8961cc43..35a31b021c 100644 --- a/src/axom/sidre/examples/sidre_createdatastore_Py.py +++ b/src/axom/sidre/examples/sidre_createdatastore_Py.py @@ -4,7 +4,7 @@ # # SPDX-License-Identifier: (BSD-3-Clause) -import pysidre +import axom.sidre as sidre import numpy as np import numpy.typing as npt @@ -13,8 +13,8 @@ # all the features in the C++ source. -def create_datastore(region: npt.NDArray[np.int_]) -> pysidre.DataStore: - ds = pysidre.DataStore() +def create_datastore(region: npt.NDArray[np.int_]) -> sidre.DataStore: + ds = sidre.DataStore() root = ds.getRoot() # Create two attributes @@ -42,10 +42,10 @@ def create_datastore(region: npt.NDArray[np.int_]) -> pysidre.DataStore: each node in a 16 x 16 x 16 hexahedron mesh. Each view is described by number of elements, offset, and stride into that data. """ - buff = ds.createBuffer(pysidre.TypeID.DOUBLE_ID, 3 * nodecount).allocate() - nodes.createView("x", buff).apply(pysidre.TypeID.DOUBLE_ID, nodecount, 0, 3) - nodes.createView("y", buff).apply(pysidre.TypeID.DOUBLE_ID, nodecount, 1, 3) - nodes.createView("z", buff).apply(pysidre.TypeID.DOUBLE_ID, nodecount, 2, 3) + buff = ds.createBuffer(sidre.TypeID.DOUBLE_ID, 3 * nodecount).allocate() + nodes.createView("x", buff).apply(sidre.TypeID.DOUBLE_ID, nodecount, 0, 3) + nodes.createView("y", buff).apply(sidre.TypeID.DOUBLE_ID, nodecount, 1, 3) + nodes.createView("z", buff).apply(sidre.TypeID.DOUBLE_ID, nodecount, 2, 3) """ Populate "fields" group @@ -55,8 +55,8 @@ def create_datastore(region: npt.NDArray[np.int_]) -> pysidre.DataStore: and stride (1). These Views could point to data associated with each of the 15 x 15 x 15 hexahedron elements defined by the nodes above. """ - temp = fields.createViewAndAllocate("temp", pysidre.TypeID.DOUBLE_ID, eltcount) - rho = fields.createViewAndAllocate("rho", pysidre.TypeID.DOUBLE_ID, eltcount) + temp = fields.createViewAndAllocate("temp", sidre.TypeID.DOUBLE_ID, eltcount) + rho = fields.createViewAndAllocate("rho", sidre.TypeID.DOUBLE_ID, eltcount) # Explicitly set values for the "vis" Attribute on the "temp" and "rho" buffers. temp.setAttributeScalar("vis", 1) @@ -69,12 +69,12 @@ def create_datastore(region: npt.NDArray[np.int_]) -> pysidre.DataStore: # numpy of int region has been passed in as a function argument. As with "temp" # and "rho", view "region" has default offset and stride. - ext.createView("region", region).apply(pysidre.TypeID.INT_ID, eltcount) + ext.createView("region", region).apply(sidre.TypeID.INT_ID, eltcount) return ds -def access_datastore(ds: pysidre.DataStore) -> pysidre.DataStore: +def access_datastore(ds: sidre.DataStore) -> sidre.DataStore: # Retrieve Group pointers root = ds.getRoot() state = root.getGroup("state") @@ -108,7 +108,7 @@ def access_datastore(ds: pysidre.DataStore) -> pysidre.DataStore: return ds -def iterate_datastore(ds: pysidre.DataStore) -> None: +def iterate_datastore(ds: sidre.DataStore) -> None: fill_line = "=" * 80 print(fill_line) diff --git a/src/axom/sidre/nanobind_sidre.cpp b/src/axom/sidre/nanobind_sidre.cpp index 9c04940034..464d30398e 100644 --- a/src/axom/sidre/nanobind_sidre.cpp +++ b/src/axom/sidre/nanobind_sidre.cpp @@ -7,14 +7,18 @@ #include #include #include +#include #include #include #include +#include #include #include "axom/config.hpp" #include "axom/core/Types.hpp" +#include "axom/slic/interface/slic.hpp" + #include "core/SidreTypes.hpp" #include "core/Buffer.hpp" #include "core/View.hpp" @@ -275,46 +279,128 @@ conduit::Node& nbObjectToNode(nb::object& o) * the ndarray is garbage collected. * * To keep Sidre's C++ semantics unchanged while making the Python API safe, - * we maintain a binding-only registry that maps a C++ View* to a copied - * nanobind::ndarray wrapper. Copying nb::ndarray increments the underlying - * ndarray owner's refcount via nanobind's internal handle, so the NumPy storage - * remains alive as long as the View exists. + * we maintain a binding-only registry of "pins": copies of the nanobind ndarray + * wrapper. Copying nb::ndarray increments the underlying ndarray owner's + * refcount via nanobind's internal handle, so the NumPy storage stays alive for + * as long as the pin exists. A pin therefore ties the external array's lifetime + * to the *C++ View's* lifetime, not to any transient Python proxy: the array + * survives even if the Python View object that set it is discarded, as long as + * the View still lives in its DataStore (see the lifetime tests). + * + * **Per-DataStore scoping.** The registry is keyed by owning DataStore* and, + * within each DataStore, by View*. This is what makes raw-pointer keys safe: + * + * - When a DataStore's Python object is collected, nanobind destroys the C++ + * DataStore (the user always holds a DataStore through Python, so the two + * lifetimes coincide). We install a weak reference on the DataStore at the + * first pin whose callback erases that DataStore's entire sub-map. Pins are + * thus released no later than DataStore destruction -- the registry never + * grows without bound, even for Views torn down by the C++ DataStore + * destructor rather than an explicit destroyView()/destroyGroup(). + * - A View* is only meaningful within its owning DataStore, and that + * DataStore's sub-map is wiped when the DataStore dies. A View* address + * reused by a *different* DataStore therefore cannot collide with a stale + * pin, and a reused DataStore* address starts from a fresh (empty) sub-map. + * - We never look up a pin by a View* that might be stale: copyView/copyGroup + * only search for the source pin while the source View is live, then re-pin + * the destination from that live ndarray value. * - * Pins are released when the external pointer is cleared (e.g. View.clear(), - * setExternalData(None)) and when views/groups are destroyed via the bound Group::destroy* APIs. + * Pins are also released eagerly when the external pointer is cleared + * (View.clear(), setExternalData(None)) and when views/groups are destroyed via + * the bound destroy* APIs, so memory is reclaimed promptly in the common case + * rather than waiting for DataStore destruction. * - * **Registry Lifetime:** The registry persists for the process lifetime and may accumulate - * entries for destroyed Views if those Views are destroyed by the C++ DataStore destructor - * rather than through the Python-wrapped destroy methods. This is acceptable because: - * (1) Dangling View* keys are never dereferenced (we only erase, never lookup by pointer) - * (2) The memory overhead is small (one map entry per external View ever created) - * (3) In typical Python usage, Views with external data are explicitly destroyed via - * destroyView()/destroyGroup(), which properly releases pins. + * \note Thread safety: all access goes through the GIL (see the module + * docstring). If the bindings ever release the GIL, this registry needs a mutex. + * + * \note Pin scoping assumes a pinned View stays within the DataStore it belonged + * to when it was pinned. Sidre reparenting (moveView/moveGroup) stays within a + * single DataStore, so a View's owning DataStore is stable for its lifetime and + * the DataStore* key never goes stale under a supported operation. + * If Sidre ever gained cross-DataStore migration of a live View, that View's pin + * would remain under its original DataStore (and be released when that DataStore is collected), + * so this invariant would need revisiting. */ -std::unordered_map>& externalDataOwnerRegistry() +struct DataStoreExternalPins +{ + std::unordered_map> pins; + // Holds the weak reference whose callback clears this sub-map; keeping it here + // keeps the callback armed for the lifetime of the entries. + nb::object datastore_weakref; +}; + +std::unordered_map& externalDataOwnerRegistry() { - // Intentionally heap-allocated so Python-owned references are not destroyed - // after interpreter finalization during static shutdown. - static auto* registry = new std::unordered_map>(); + // Intentionally heap-allocated so any Python-owned references it still holds + // are not destroyed after interpreter finalization during static shutdown. + static auto* registry = new std::unordered_map(); return *registry; } -template -void pinExternalDataOwner(View* view, const nb::ndarray& owner) +//! Return the owning DataStore of a View, or nullptr if it has none yet. +DataStore* owningDataStore(View* view) { - if(view != nullptr) + if(view == nullptr) + { + return nullptr; + } + Group* group = view->getOwningGroup(); + return (group != nullptr) ? group->getDataStore() : nullptr; +} + +//! Erase all pins recorded for \a ds (called when the DataStore is collected). +void releaseDataStoreExternalPins(DataStore* ds) { externalDataOwnerRegistry().erase(ds); } + +/*! + * \brief Record \a owner as the pin for \a view, scoped to its DataStore. + * + * On the first pin into a given DataStore, installs a weak reference on the + * DataStore's Python object so the sub-map is cleared when the DataStore is + * destroyed. Re-assigning a View*'s pin releases the previous ndarray wrapper. + */ +void pinExternalDataOwner(View* view, const nb::ndarray<>& owner) +{ + DataStore* ds = owningDataStore(view); + // Enforce the precondition in debug builds; + // release builds fall through to the null-safe early return below. + SLIC_ASSERT_MSG(view == nullptr || ds != nullptr, + "pinExternalDataOwner: a non-null View is expected to have an owning DataStore"); + if(view == nullptr || ds == nullptr) + { + return; + } + + DataStoreExternalPins& entry = externalDataOwnerRegistry()[ds]; + if(!entry.datastore_weakref.is_valid()) { - // Note: Map assignment automatically releases the previous ndarray wrapper if present. - // When nb::ndarray<> is destroyed, nanobind decrements the underlying Python object's refcount - externalDataOwnerRegistry()[view] = nb::ndarray<>(owner); + // Retrieve the DataStore's existing Python wrapper and attach a weakref + // whose callback clears this DataStore's pins. nb::find returns a null + // object if no wrapper exists; in that (unexpected) case we simply skip the + // weakref -- the eager release paths still apply, and the worst case is the + // pre-existing process-lifetime retention. + nb::object ds_obj = nb::find(*ds); + if(ds_obj.is_valid() && !ds_obj.is_none()) + { + entry.datastore_weakref = + nb::weakref(ds_obj, nb::cpp_function([ds](nb::handle) { releaseDataStoreExternalPins(ds); })); + } } + + // Map assignment releases the previous ndarray wrapper if one was present. + entry.pins[view] = nb::ndarray<>(owner); } void releaseExternalDataOwner(View* view) { - if(view != nullptr) + DataStore* ds = owningDataStore(view); + if(view == nullptr || ds == nullptr) + { + return; + } + auto it = externalDataOwnerRegistry().find(ds); + if(it != externalDataOwnerRegistry().end()) { - externalDataOwnerRegistry().erase(view); + it->second.pins.erase(view); } } @@ -345,14 +431,17 @@ void releaseExternalDataOwnersOfViews(Group& group) } /*! - * \brief Copy external data pin from source View to destination View. + * \brief Copy the external-data pin from a source View to a destination View. * - * When copyView() creates a shallow copy that shares external data, the new View - * needs its own pin to prevent the numpy array from being garbage collected. - * This function looks up the source View's pin and copies it to the destination. + * copyView() makes a shallow copy that shares the external pointer, so the + * destination needs its own pin to keep the NumPy array alive independently of + * the source. The source View is live for the duration of the copy (the caller + * holds it), so looking up its pin within its own DataStore's sub-map is safe; + * we then pin the destination from that live ndarray value. We never search the + * registry by a View* that could be stale. * - * \param src_view Source View (must have an external data pin) - * \param dst_view Destination View (will receive a copy of the pin) + * \param src_view Source View (live; expected to hold an external data pin) + * \param dst_view Destination View (receives a copy of the pin) */ void copyExternalDataOwner(const View* src_view, View* dst_view) { @@ -361,12 +450,25 @@ void copyExternalDataOwner(const View* src_view, View* dst_view) return; } + DataStore* src_ds = owningDataStore(const_cast(src_view)); + if(src_ds == nullptr) + { + return; + } + auto& registry = externalDataOwnerRegistry(); - auto it = registry.find(const_cast(src_view)); - if(it != registry.end()) + auto ds_it = registry.find(src_ds); + if(ds_it == registry.end()) + { + return; + } + + auto pin_it = ds_it->second.pins.find(const_cast(src_view)); + if(pin_it != ds_it->second.pins.end()) { - // Copy the ndarray handle to the new View, incrementing its refcount - registry[dst_view] = it->second; + // Re-pin the destination from the source's live ndarray value (scoped to + // the destination's own DataStore by pinExternalDataOwner). + pinExternalDataOwner(dst_view, pin_it->second); } } @@ -463,9 +565,9 @@ MPI_Comm mpiCommFromObject(nb::object comm) * duplicate or free it. That borrowed-communicator contract works for C++ * callers, but it is unsafe for mpi4py objects: py2f() exposes the object's * current communicator handle, and Python code may later drop or explicitly - * Free() that object while pysidre.IOManager is still alive. + * Free() that object while axom.sidre.IOManager is still alive. * - * PyIOManager keeps the public Python class name as pysidre.IOManager while + * PyIOManager keeps the public Python class name as axom.sidre.IOManager while * giving the binding its own lifetime boundary. It duplicates the input * communicator, constructs sidre::IOManager with that duplicate, destroys the * IOManager first, and then frees the duplicate when MPI is still active. @@ -476,7 +578,8 @@ class PyIOManager PyIOManager(MPI_Comm comm, bool use_scr) { int err = MPI_Comm_dup(comm, &m_comm); - SLIC_ERROR_IF(err != MPI_SUCCESS, "Failed to duplicate MPI communicator for pysidre.IOManager"); + SLIC_ERROR_IF(err != MPI_SUCCESS, + "Failed to duplicate MPI communicator for axom.sidre.IOManager"); m_manager = std::make_unique(m_comm, use_scr); } @@ -511,7 +614,9 @@ class PyIOManager }; #endif -NB_MODULE(pysidre, m_sidre) +// The extension installs as ``axom/sidre/_sidre..so`` and is re-exported +// by the ``axom.sidre`` package (see src/python/src/axom/sidre/__init__.py). +NB_MODULE(_sidre, m_sidre) { m_sidre.doc() = R"pbdoc( A python extension for Axom's Sidre component. @@ -529,10 +634,16 @@ NB_MODULE(pysidre, m_sidre) **External Data Lifetime:** Views can reference external numpy arrays via setExternalData() or createView(). The binding automatically pins these arrays to prevent garbage collection while - the View exists. Pins are released when: - - View.clear() is called - - The View is destroyed via destroyView() or destroyViewAndData() + the View exists, so an array stays valid even if the Python View object that set + it is discarded (as long as the View still lives in its DataStore). Pins are + scoped per-DataStore and are released when: + - View.clear() or setExternalData(None) is called + - The View is destroyed via destroyView() / destroyViewAndData() - The owning Group hierarchy is destroyed via destroyGroup*() methods + - The owning DataStore is destroyed (a weak reference on the DataStore clears + its remaining pins, so the registry never grows without bound -- even for + Views torn down by the C++ DataStore destructor rather than an explicit + destroy* call) **Reallocation Hazards:** Arrays obtained via getDataArray() are zero-copy views into Sidre storage. @@ -604,7 +715,10 @@ NB_MODULE(pysidre, m_sidre) bindIterator(m_sidre, "ViewIterator"); // Bindings for the DataStore class - nb::class_(m_sidre, "DataStore") + // DataStore is weak-referenceable so the external-data registry can attach a + // weakref callback that releases that DataStore's pins when it is destroyed + // (see externalDataOwnerRegistry). + nb::class_(m_sidre, "DataStore", nb::is_weak_referenceable()) .def(nb::init<>()) .def("getRoot", nb::overload_cast<>(&DataStore::getRoot), @@ -750,7 +864,13 @@ NB_MODULE(pysidre, m_sidre) .def("getIndex", &Buffer::getIndex, "Return the unique index of this Buffer object.") .def("getNumViews", &Buffer::getNumViews, "Return number of Views this Buffer is attached to.") // .def("getVoidPtr", &Buffer::getVoidPtr, "Return void-pointer to data held by Buffer.") - .def("getDataArray", &bufferToNumpyArray, "Return the data held by the Buffer as a numpy array.") + .def("getDataArray", + &bufferToNumpyArray, + "Return the data held by the Buffer as a numpy array.\n\n" + "The array is a zero-copy view into the Buffer's storage " + "and keeps the Buffer (and its DataStore) alive while referenced. " + "Buffer.reallocate() can move the storage, leaving a previously returned array " + "pointing at freed memory; re-acquire the array after any reallocation.") .def("getTypeID", &Buffer::getTypeID, "Return type of data owned by this Buffer object.") .def("getNumElements", &Buffer::getNumElements, @@ -937,26 +1057,23 @@ NB_MODULE(pysidre, m_sidre) nb::arg("allocID") = INVALID_ALLOCATOR_ID) .def( "setExternalData", - [](View& self, nb::object external_ptr) { - if(external_ptr.is_none()) + [](View& self, std::optional> external_ptr) { + // A single undescribed-data overload covering both None + // (clear the external pointer and release any pin) and a numpy array (set + pin). + // Using std::optional lets nanobind reject a non-array argument + // with a clean "incompatible function arguments" error + // rather than throwing mid-body from an explicit cast. + if(!external_ptr.has_value()) { View* result = self.setExternalDataPtr(nullptr); releaseExternalDataOwner(&self); return result; } - nb::ndarray<> owner = nb::cast>(external_ptr); - return setExternalDataAndPinOwner(self, owner); + return setExternalDataAndPinOwner(self, *external_ptr); }, nb::rv_policy::reference, - "Set the View to hold undescribed external data (numpy array).", + "Set the View to hold undescribed external data, or clear it when passed None.", nb::arg("external_ptr").none()) - .def( - "setExternalData", - [](View& self, const nb::ndarray<>& external_ptr) { - return setExternalDataAndPinOwner(self, external_ptr); - }, - nb::rv_policy::reference, - "Set the View to hold undescribed external data (numpy array).") .def( "setExternalData", [](View& self, TypeID type, IndexType num_elems, const nb::ndarray<>& external_ptr) { @@ -980,7 +1097,14 @@ NB_MODULE(pysidre, m_sidre) &View::getString, nb::rv_policy::reference, "Return the string contained in the View.") - .def("getDataArray", &viewToNumpyArray, "Return the data held by the View as a numpy array.") + .def("getDataArray", + &viewToNumpyArray, + "Return the data held by the View as a numpy array.\n\n" + "The array is a zero-copy view into the View's storage " + "and keeps the View (and its DataStore) alive while referenced. " + "View.reallocate() (or reallocating the underlying Buffer) can move the storage, " + "leaving a previously returned array pointing at freed memory; " + "re-acquire the array after any reallocation.") .def( "getDataInt", diff --git a/src/axom/sidre/tests/CMakeLists.txt b/src/axom/sidre/tests/CMakeLists.txt index 8b34e2648b..6378c5a795 100644 --- a/src/axom/sidre/tests/CMakeLists.txt +++ b/src/axom/sidre/tests/CMakeLists.txt @@ -58,6 +58,7 @@ set(python_sidre_tests sidre_external_Py.py sidre_attribute_Py.py sidre_lifetime_Py.py + sidre_import_Py.py ) set(sidre_gtests_depends_on sidre fmt gtest) diff --git a/src/axom/sidre/tests/sidre_attribute_Py.py b/src/axom/sidre/tests/sidre_attribute_Py.py index 0867ef1e01..ac92fc0fd5 100644 --- a/src/axom/sidre/tests/sidre_attribute_Py.py +++ b/src/axom/sidre/tests/sidre_attribute_Py.py @@ -4,7 +4,7 @@ # # SPDX-License-Identifier: (BSD-3-Clause) -import pysidre +import axom.sidre as sidre import numpy as np import conduit @@ -34,7 +34,7 @@ # Python equivalent of nullptr g_attr_null = None -if pysidre.AXOM_USE_HDF5: +if sidre.AXOM_USE_HDF5: g_nprotocols = 3 g_protocols = ["sidre_json", "sidre_hdf5", "json"] else: @@ -51,7 +51,7 @@ def test_create_attr(): print("Some warnings are expected in the 'create_attr' test") - ds = pysidre.DataStore() + ds = sidre.DataStore() nattrs = ds.getNumAttributes() assert nattrs == 0 @@ -64,7 +64,7 @@ def test_create_attr(): # Create string attribute color = ds.createAttributeString(g_name_color, g_color_none) assert color is not None - assert color.getTypeID() == pysidre.TypeID.CHAR8_STR_ID + assert color.getTypeID() == sidre.TypeID.CHAR8_STR_ID attr_index = color.getIndex() assert attr_index == 0 @@ -141,7 +141,7 @@ def test_create_attr(): def test_view_attr(): print("Some warnings are expected in the 'view_attr' test") - ds = pysidre.DataStore() + ds = sidre.DataStore() # Create all attributes for DataStore attr_color = ds.createAttributeString(g_name_color, g_color_none) @@ -268,16 +268,16 @@ def test_view_attr(): def test_view_int_and_double(): print("Some warnings are expected in the 'view_int_and_double' test") - ds = pysidre.DataStore() + ds = sidre.DataStore() # Create all attributes for DataStore attr_dump = ds.createAttributeScalar(g_name_dump, g_dump_no) assert attr_dump is not None - assert attr_dump.getTypeID() == pysidre.TypeID.INT32_ID + assert attr_dump.getTypeID() == sidre.TypeID.INT32_ID attr_size = ds.createAttributeScalar(g_name_size, g_size_small) assert attr_size is not None - assert attr_size.getTypeID() == pysidre.TypeID.FLOAT64_ID + assert attr_size.getTypeID() == sidre.TypeID.FLOAT64_ID root = ds.getRoot() @@ -328,16 +328,16 @@ def test_view_int_and_double(): def test_set_default(): print("Some warnings are expected in the 'set_default' test") - ds = pysidre.DataStore() + ds = sidre.DataStore() # Create all attributes for DataStore attr_dump = ds.createAttributeScalar(g_name_dump, g_dump_no) assert attr_dump is not None - assert attr_dump.getTypeID() == pysidre.TypeID.INT32_ID + assert attr_dump.getTypeID() == sidre.TypeID.INT32_ID attr_size = ds.createAttributeScalar(g_name_size, g_size_small) assert attr_size is not None - assert attr_size.getTypeID() == pysidre.TypeID.FLOAT64_ID + assert attr_size.getTypeID() == sidre.TypeID.FLOAT64_ID root = ds.getRoot() @@ -383,7 +383,7 @@ def test_set_default(): def test_as_node(): print("Some warnings are expected in the 'as_node' test") - ds = pysidre.DataStore() + ds = sidre.DataStore() # Create attributes for DataStore attr_color = ds.createAttributeString(g_name_color, g_color_none) @@ -418,7 +418,7 @@ def test_as_node(): def test_overloads(): print("Some warnings are expected in the 'overloads' test") - ds = pysidre.DataStore() + ds = sidre.DataStore() # Create string and scalar attributes attr_color = ds.createAttributeString(g_name_color, g_color_none) @@ -497,12 +497,12 @@ def test_overloads(): # Check some errors assert view.getAttributeScalarInt(g_attr_null) == 0 - assert view.getAttributeScalarInt(pysidre.InvalidIndex) == 0 + assert view.getAttributeScalarInt(sidre.InvalidIndex) == 0 assert view.getAttributeScalarInt("noname") == 0 def test_loop_attributes(): - ds = pysidre.DataStore() + ds = sidre.DataStore() # Create attributes for DataStore color = ds.createAttributeString(g_name_color, g_color_none) @@ -525,9 +525,9 @@ def test_loop_attributes(): idx3 = ds.getNextValidAttributeIndex(idx2) assert idx3 == 2 idx4 = ds.getNextValidAttributeIndex(idx3) - assert idx4 == pysidre.InvalidIndex + assert idx4 == sidre.InvalidIndex idx5 = ds.getNextValidAttributeIndex(idx4) - assert idx5 == pysidre.InvalidIndex + assert idx5 == sidre.InvalidIndex # ---------------------------------------- root = ds.getRoot() @@ -545,7 +545,7 @@ def test_loop_attributes(): idx3 = view1.getNextValidAttrValueIndex(idx2) assert idx3 == 2 idx4 = view1.getNextValidAttrValueIndex(idx3) - assert idx4 == pysidre.InvalidIndex + assert idx4 == sidre.InvalidIndex # set first attribute view2 = root.createView("view2") @@ -554,7 +554,7 @@ def test_loop_attributes(): idx1 = view2.getFirstValidAttrValueIndex() assert idx1 == 0 idx2 = view2.getNextValidAttrValueIndex(idx1) - assert idx2 == pysidre.InvalidIndex + assert idx2 == sidre.InvalidIndex # set last attribute view3 = root.createView("view3") @@ -563,7 +563,7 @@ def test_loop_attributes(): idx1 = view3.getFirstValidAttrValueIndex() assert idx1 == 2 idx2 = view3.getNextValidAttrValueIndex(idx1) - assert idx2 == pysidre.InvalidIndex + assert idx2 == sidre.InvalidIndex # set first and last attributes view4 = root.createView("view4") @@ -575,19 +575,19 @@ def test_loop_attributes(): idx2 = view4.getNextValidAttrValueIndex(idx1) assert idx2 == 2 idx3 = view4.getNextValidAttrValueIndex(idx2) - assert idx3 == pysidre.InvalidIndex + assert idx3 == sidre.InvalidIndex # no attributes view5 = root.createView("view5") idx1 = view5.getFirstValidAttrValueIndex() - assert idx1 == pysidre.InvalidIndex + assert idx1 == sidre.InvalidIndex idx2 = view5.getNextValidAttrValueIndex(idx1) - assert idx2 == pysidre.InvalidIndex + assert idx2 == sidre.InvalidIndex def test_iterate_attributes(): - ds = pysidre.DataStore() + ds = sidre.DataStore() # Create attributes for DataStore color = ds.createAttributeString(g_name_color, g_color_none) @@ -627,7 +627,7 @@ def test_save_attributes(): idata = np.zeros(5, dtype=int) file_path_base = "sidre_attribute_datastore_" - ds1 = pysidre.DataStore() + ds1 = sidre.DataStore() root1 = ds1.getRoot() # Create attributes for DataStore @@ -649,13 +649,13 @@ def test_save_attributes(): view1a.setAttributeScalar(size, g_size_small) # buffer - view1b = root1.createViewAndAllocate("buffer", pysidre.TypeID.INT_ID, 5) + view1b = root1.createViewAndAllocate("buffer", sidre.TypeID.INT_ID, 5) bdata = view1b.getDataArray() view1b.setAttributeString(color, "color-buffer") view1b.setAttributeScalar(size, g_size_medium) # external - view1c = root1.createView("external", pysidre.TypeID.INT_ID, 5, idata) + view1c = root1.createView("external", sidre.TypeID.INT_ID, 5, idata) view1c.setAttributeScalar(size, g_size_large) # scalar @@ -692,7 +692,7 @@ def test_save_attributes(): file_path = file_path_base + g_protocols[i] - ds2 = pysidre.DataStore() + ds2 = sidre.DataStore() root2 = ds2.getRoot() root2.load(file_path, g_protocols[i]) @@ -757,7 +757,7 @@ def test_save_by_attribute(): jdata = np.zeros(5, dtype=int) file_path_base = "sidre_attribute_by_attribute_" - ds1 = pysidre.DataStore() + ds1 = sidre.DataStore() root1 = ds1.getRoot() # Create attributes for DataStore @@ -772,9 +772,8 @@ def test_save_by_attribute(): root1.createViewScalar("grp1a/grp1b/view3", 3) root1.createViewScalar("grp2a/view4", 4) # make sure empty "views" not saved root1.createViewScalar("grp2a/grp2b/view5", 5).setAttributeScalar(dump, g_dump_yes) - root1.createView("view6", pysidre.TypeID.INT32_ID, 5, - idata).setAttributeScalar(dump, g_dump_yes) - root1.createView("grp3a/grp3b/view7", pysidre.TypeID.INT32_ID, 5, jdata) + root1.createView("view6", sidre.TypeID.INT32_ID, 5, idata).setAttributeScalar(dump, g_dump_yes) + root1.createView("grp3a/grp3b/view7", sidre.TypeID.INT32_ID, 5, jdata) for i in range(5): idata[i] = i @@ -798,7 +797,7 @@ def test_save_by_attribute(): continue file_path = file_path_base + g_protocols[i] - ds2 = pysidre.DataStore() + ds2 = sidre.DataStore() root2 = ds2.getRoot() root2.load(file_path, g_protocols[i]) @@ -821,7 +820,7 @@ def test_save_load_group_with_attributes_new_ds(): filename = f"saveFile_{protocol}.{ext}" # Set up first datastore and save to disk - ds1 = pysidre.DataStore() + ds1 = sidre.DataStore() ds1.createAttributeScalar("attr", 10) ds1.createAttributeString(g_name_color, g_color_none) @@ -831,9 +830,9 @@ def test_save_load_group_with_attributes_new_ds(): gr1.createViewScalar("scalar3", 3).setAttributeString(g_name_color, g_color_blue) assert ds1.getNumAttributes() == 2 - assert (pysidre.TypeID.INT32_ID == ds1.getAttribute("attr").getTypeID() - or pysidre.TypeID.INT64_ID == ds1.getAttribute("attr").getTypeID()) - assert pysidre.TypeID.CHAR8_STR_ID == ds1.getAttribute(g_name_color).getTypeID() + assert (sidre.TypeID.INT32_ID == ds1.getAttribute("attr").getTypeID() + or sidre.TypeID.INT64_ID == ds1.getAttribute("attr").getTypeID()) + assert sidre.TypeID.CHAR8_STR_ID == ds1.getAttribute(g_name_color).getTypeID() assert not gr1.getView("scalar1").hasAttributeValue(g_name_color) @@ -852,14 +851,14 @@ def test_save_load_group_with_attributes_new_ds(): continue # Load second datastore from saved data - ds2 = pysidre.DataStore() + ds2 = sidre.DataStore() gr2 = ds2.getRoot().createGroup("gr") gr2.load(filename, protocol) assert ds2.getNumAttributes() == 2 - assert (pysidre.TypeID.INT32_ID == ds2.getAttribute("attr").getTypeID() - or pysidre.TypeID.INT64_ID == ds2.getAttribute("attr").getTypeID()) - assert pysidre.TypeID.CHAR8_STR_ID == ds2.getAttribute(g_name_color).getTypeID() + assert (sidre.TypeID.INT32_ID == ds2.getAttribute("attr").getTypeID() + or sidre.TypeID.INT64_ID == ds2.getAttribute("attr").getTypeID()) + assert sidre.TypeID.CHAR8_STR_ID == ds2.getAttribute(g_name_color).getTypeID() assert gr2.hasView("scalar1") assert not gr2.getView("scalar1").hasAttributeValue(g_name_color) @@ -891,7 +890,7 @@ def test_save_load_group_with_attributes_same_ds(): print(f"Checking attribute save/load w/ protocol '{protocol}' using file '{filename}'") # Create the DataStore and attributes - ds = pysidre.DataStore() + ds = sidre.DataStore() ds.createAttributeScalar("attr", 10) ds.createAttributeString(g_name_color, g_color_none) diff --git a/src/axom/sidre/tests/sidre_buffer_Py.py b/src/axom/sidre/tests/sidre_buffer_Py.py index a42db76480..5b3ee76848 100644 --- a/src/axom/sidre/tests/sidre_buffer_Py.py +++ b/src/axom/sidre/tests/sidre_buffer_Py.py @@ -4,14 +4,14 @@ # # SPDX-License-Identifier: (BSD-3-Clause) -import pysidre +import axom.sidre as sidre import numpy as np NUM_BYTES_INT_32 = 4 def test_create_buffers(): - ds = pysidre.DataStore() + ds = sidre.DataStore() assert ds.getNumBuffers() == 0 dbuff_0 = ds.createBuffer() @@ -34,17 +34,17 @@ def test_create_buffers(): def test_alloc_buffer_for_int_array(): - ds = pysidre.DataStore() + ds = sidre.DataStore() dbuff = ds.createBuffer() elem_count = 10 - dbuff.allocate(pysidre.TypeID.INT32_ID, elem_count) + dbuff.allocate(sidre.TypeID.INT32_ID, elem_count) # Should be a warning and no-op, buffer is already allocated, we don't want # to re-allocate and leak memory. dbuff.allocate() - assert dbuff.getTypeID() == pysidre.TypeID.INT32_ID + assert dbuff.getTypeID() == sidre.TypeID.INT32_ID assert dbuff.getNumElements() == elem_count assert dbuff.getTotalBytes() == NUM_BYTES_INT_32 * elem_count @@ -65,12 +65,12 @@ def test_alloc_buffer_for_int_array(): def test_init_buffer_for_int_array(): elem_count = 10 - ds = pysidre.DataStore() + ds = sidre.DataStore() dbuff = ds.createBuffer() - dbuff.allocate(pysidre.TypeID.INT32_ID, elem_count) + dbuff.allocate(sidre.TypeID.INT32_ID, elem_count) - assert dbuff.getTypeID() == pysidre.TypeID.INT32_ID + assert dbuff.getTypeID() == sidre.TypeID.INT32_ID assert dbuff.getNumElements() == elem_count assert dbuff.getTotalBytes() == NUM_BYTES_INT_32 * elem_count @@ -92,12 +92,12 @@ def test_realloc_buffer(): orig_elem_count = 5 mod_elem_count = 10 - ds = pysidre.DataStore() + ds = sidre.DataStore() dbuff = ds.createBuffer() - dbuff.allocate(pysidre.TypeID.INT32_ID, orig_elem_count) + dbuff.allocate(sidre.TypeID.INT32_ID, orig_elem_count) - assert dbuff.getTypeID() == pysidre.TypeID.INT32_ID + assert dbuff.getTypeID() == sidre.TypeID.INT32_ID assert dbuff.getNumElements() == orig_elem_count assert dbuff.getTotalBytes() == NUM_BYTES_INT_32 * orig_elem_count @@ -111,7 +111,7 @@ def test_realloc_buffer(): dbuff.reallocate(mod_elem_count) - assert dbuff.getTypeID() == pysidre.TypeID.INT32_ID + assert dbuff.getTypeID() == sidre.TypeID.INT32_ID assert dbuff.getNumElements() == mod_elem_count assert dbuff.getTotalBytes() == NUM_BYTES_INT_32 * mod_elem_count diff --git a/src/axom/sidre/tests/sidre_datastore_unit_Py.py b/src/axom/sidre/tests/sidre_datastore_unit_Py.py index d90f0b5b38..fed76f9185 100644 --- a/src/axom/sidre/tests/sidre_datastore_unit_Py.py +++ b/src/axom/sidre/tests/sidre_datastore_unit_Py.py @@ -4,7 +4,7 @@ # # SPDX-License-Identifier: (BSD-3-Clause) -import pysidre +import axom.sidre as sidre import random @@ -16,20 +16,20 @@ def verify_empty_group_named(dg, name): assert not dg.hasGroup(0) assert not dg.hasGroup(1) assert not dg.hasGroup("some_name") - assert dg.getGroupIndex("some_other_name") == pysidre.InvalidIndex - assert dg.getFirstValidGroupIndex() == pysidre.InvalidIndex - assert dg.getNextValidGroupIndex(0) == pysidre.InvalidIndex - assert dg.getNextValidGroupIndex(4) == pysidre.InvalidIndex + assert dg.getGroupIndex("some_other_name") == sidre.InvalidIndex + assert dg.getFirstValidGroupIndex() == sidre.InvalidIndex + assert dg.getNextValidGroupIndex(0) == sidre.InvalidIndex + assert dg.getNextValidGroupIndex(4) == sidre.InvalidIndex assert dg.getNumViews() == 0 assert not dg.hasView(-1) assert not dg.hasView(0) assert not dg.hasView(1) assert not dg.hasView("some_name") - assert dg.getViewIndex("some_other_name") == pysidre.InvalidIndex - assert dg.getFirstValidViewIndex() == pysidre.InvalidIndex - assert dg.getNextValidViewIndex(0) == pysidre.InvalidIndex - assert dg.getNextValidViewIndex(4) == pysidre.InvalidIndex + assert dg.getViewIndex("some_other_name") == sidre.InvalidIndex + assert dg.getFirstValidViewIndex() == sidre.InvalidIndex + assert dg.getNextValidViewIndex(0) == sidre.InvalidIndex + assert dg.getNextValidViewIndex(4) == sidre.InvalidIndex def verify_buffer_identity(ds, bs): @@ -41,7 +41,7 @@ def verify_buffer_identity(ds, bs): # Does ds contain the buffer IDs and pointers we expect? iterated_count = 0 idx = ds.getFirstValidBufferIndex() - while idx != pysidre.InvalidIndex and iterated_count < bufcount: + while idx != sidre.InvalidIndex and iterated_count < bufcount: assert idx in bs if idx in bs: assert bs[idx] == ds.getBuffer(idx) @@ -50,11 +50,11 @@ def verify_buffer_identity(ds, bs): # Have we iterated over exactly the number of buffers we expect, finishing on InvalidIndex? assert iterated_count == bufcount - assert idx == pysidre.InvalidIndex + assert idx == sidre.InvalidIndex def test_default_ctor(): - ds = pysidre.DataStore() + ds = sidre.DataStore() # After construction, the DataStore should contain no buffers. assert ds.getNumBuffers() == 0 @@ -64,9 +64,9 @@ def test_default_ctor(): assert not ds.hasBuffer(1) assert not ds.hasBuffer(8) - assert ds.getFirstValidBufferIndex() == pysidre.InvalidIndex - assert ds.getNextValidBufferIndex(0) == pysidre.InvalidIndex - assert ds.getNextValidBufferIndex(4) == pysidre.InvalidIndex + assert ds.getFirstValidBufferIndex() == sidre.InvalidIndex + assert ds.getNextValidBufferIndex(0) == sidre.InvalidIndex + assert ds.getNextValidBufferIndex(4) == sidre.InvalidIndex # The new DataStore should contain exactly one group, the root group. # The root group should be named "" and should contain no views and no groups. @@ -82,7 +82,7 @@ def test_default_ctor(): # The dtor destroys all buffers and deletes the root group. # An outside tool should be used to check for proper memory cleanup. def test_create_destroy_buffers_basic(): - ds = pysidre.DataStore() + ds = sidre.DataStore() assert ds.getNumBuffers() == 0 # Basic tests @@ -92,7 +92,7 @@ def test_create_destroy_buffers_basic(): buffer_index = ds.getFirstValidBufferIndex() assert dbuff.getIndex() == 0 assert dbuff.getIndex() == buffer_index - assert ds.getNextValidBufferIndex(buffer_index) == pysidre.InvalidIndex + assert ds.getNextValidBufferIndex(buffer_index) == sidre.InvalidIndex # Do we get the buffer we expect? assert dbuff == ds.getBuffer(buffer_index) @@ -102,14 +102,14 @@ def test_create_destroy_buffers_basic(): ds.destroyBuffer(buffer_index) # should be no buffers assert ds.getNumBuffers() == 0 - assert ds.getFirstValidBufferIndex() == pysidre.InvalidIndex + assert ds.getFirstValidBufferIndex() == sidre.InvalidIndex assert not ds.hasBuffer(buffer_index) assert ds.getBuffer(buffer_index) is None assert ds.getBuffer(bad_buffer_index) is None def test_create_destroy_buffers_order(): - ds = pysidre.DataStore() + ds = sidre.DataStore() assert ds.getNumBuffers() == 0 dbuff = ds.createBuffer() @@ -119,7 +119,7 @@ def test_create_destroy_buffers_order(): ds.destroyBuffer(dbuff) # After destroy, test that buffer index should be available again for reuse. - dbuff2 = ds.createBuffer(pysidre.TypeID.FLOAT32_ID, 16) + dbuff2 = ds.createBuffer(sidre.TypeID.FLOAT32_ID, 16) d2_index = dbuff2.getIndex() assert ds.getFirstValidBufferIndex() == buffer_index assert d2_index == buffer_index @@ -164,7 +164,7 @@ def test_create_destroy_buffers_order(): def test_create_destroy_buffers_views(): - ds = pysidre.DataStore() + ds = sidre.DataStore() assert ds.getNumBuffers() == 0 dbuff3 = ds.createBuffer() @@ -253,39 +253,39 @@ def irhall(n): # Test iteration through buffers, as well as proper index and buffer behavior # while buffers are created and deleted def test_iterate_buffers_basic(): - ds = pysidre.DataStore() + ds = sidre.DataStore() assert ds.getNumBuffers() == 0 bad_buffer_index = 9999 # Do we get sidre::InvalidIndex for several queries with no buffers? - assert ds.getFirstValidBufferIndex() == pysidre.InvalidIndex - assert ds.getNextValidBufferIndex(0) == pysidre.InvalidIndex - assert ds.getNextValidBufferIndex(bad_buffer_index) == pysidre.InvalidIndex - assert ds.getNextValidBufferIndex(pysidre.InvalidIndex) == pysidre.InvalidIndex + assert ds.getFirstValidBufferIndex() == sidre.InvalidIndex + assert ds.getNextValidBufferIndex(0) == sidre.InvalidIndex + assert ds.getNextValidBufferIndex(bad_buffer_index) == sidre.InvalidIndex + assert ds.getNextValidBufferIndex(sidre.InvalidIndex) == sidre.InvalidIndex # Create one data buffer, verify its index is zero, and that iterators behave as expected initial = ds.createBuffer() assert initial.getIndex() == 0 assert ds.getNumBuffers() == 1 assert ds.getFirstValidBufferIndex() == 0 - assert ds.getNextValidBufferIndex(0) == pysidre.InvalidIndex + assert ds.getNextValidBufferIndex(0) == sidre.InvalidIndex # Destroy the data buffer, verify that iterators behave as expected ds.destroyBuffer(initial) assert ds.getNumBuffers() == 0 - assert ds.getFirstValidBufferIndex() == pysidre.InvalidIndex - assert ds.getNextValidBufferIndex(0) == pysidre.InvalidIndex + assert ds.getFirstValidBufferIndex() == sidre.InvalidIndex + assert ds.getNextValidBufferIndex(0) == sidre.InvalidIndex def test_iterate_buffers_simple(): - ds = pysidre.DataStore() + ds = sidre.DataStore() assert ds.getNumBuffers() == 0 bs = {} bufcount = 20 for i in range(bufcount): - b = ds.createBuffer(pysidre.TypeID.FLOAT64_ID, 400 * i) + b = ds.createBuffer(sidre.TypeID.FLOAT64_ID, 400 * i) idx = b.getIndex() bs[idx] = b @@ -293,14 +293,14 @@ def test_iterate_buffers_simple(): def test_iterate_buffers_iterators(): - ds = pysidre.DataStore() + ds = sidre.DataStore() assert ds.getNumBuffers() == 0 bs = {} bufcount = 20 for i in range(bufcount): - b = ds.createBuffer(pysidre.TypeID.FLOAT64_ID, 400 * i) + b = ds.createBuffer(sidre.TypeID.FLOAT64_ID, 400 * i) idx = b.getIndex() bs[idx] = b @@ -308,7 +308,7 @@ def test_iterate_buffers_iterators(): for buff in ds.buffers(): idx = buff.getIndex() found_buffers += 1 - assert pysidre.indexIsValid(idx) + assert sidre.indexIsValid(idx) assert ds.getBuffer(idx) == buff assert bs[idx] == buff assert found_buffers == bufcount @@ -316,7 +316,7 @@ def test_iterate_buffers_iterators(): # Test creating and allocating buffers, then destroying several of them def test_create_delete_buffers_iterate(): - ds = pysidre.DataStore() + ds = sidre.DataStore() assert ds.getNumBuffers() == 0 bs = {} @@ -324,7 +324,7 @@ def test_create_delete_buffers_iterate(): # Initially, create some buffers of varying size for i in range(bufcount): - b = ds.createBuffer(pysidre.TypeID.FLOAT64_ID, (400 * i) % 10000) + b = ds.createBuffer(sidre.TypeID.FLOAT64_ID, (400 * i) % 10000) b.allocate() idx = b.getIndex() bs[idx] = b @@ -341,12 +341,12 @@ def test_create_delete_buffers_iterate(): def test_iterate_buffers_with_delete_iterators(): - ds = pysidre.DataStore() + ds = sidre.DataStore() assert ds.getNumBuffers() == 0 init_buff_count = 22 for i in range(init_buff_count): - ds.createBuffer(pysidre.TypeID.FLOAT64_ID, 400 * i) + ds.createBuffer(sidre.TypeID.FLOAT64_ID, 400 * i) assert ds.getNumBuffers() == init_buff_count # Remove a few buffers by index @@ -358,7 +358,7 @@ def test_iterate_buffers_with_delete_iterators(): # Add a buffer, expect it to reuse a lower index assert not ds.hasBuffer(5) - buff = ds.createBuffer(pysidre.TypeID.FLOAT64_ID, 10) + buff = ds.createBuffer(sidre.TypeID.FLOAT64_ID, 10) idx = buff.getIndex() assert idx < init_buff_count assert ds.hasBuffer(idx) @@ -378,14 +378,14 @@ def test_iterate_buffers_with_delete_iterators(): for buff in ds.buffers(): idx = buff.getIndex() found_buffers += 1 - assert pysidre.indexIsValid(idx) + assert sidre.indexIsValid(idx) assert ds.getBuffer(idx) == buff assert found_buffers == exp_buff_count # Test creating+allocating buffers, then destroying several of them, repeatedly def test_loop_create_delete_buffers_iterate(): - ds = pysidre.DataStore() + ds = sidre.DataStore() assert ds.getNumBuffers() == 0 bs = {} @@ -394,7 +394,7 @@ def test_loop_create_delete_buffers_iterate(): # Initially, create some buffers of varying size for i in range(initbufcount): - b = ds.createBuffer(pysidre.TypeID.FLOAT64_ID, (400 * i) % 10000) + b = ds.createBuffer(sidre.TypeID.FLOAT64_ID, (400 * i) % 10000) b.allocate() idx = b.getIndex() bs[idx] = b @@ -422,7 +422,7 @@ def test_loop_create_delete_buffers_iterate(): elif delta > 0: addcount = delta for _ in range(addcount): - buf = ds.createBuffer(pysidre.TypeID.FLOAT64_ID, 400) + buf = ds.createBuffer(sidre.TypeID.FLOAT64_ID, 400) buf.allocate() addid = buf.getIndex() assert ds.hasBuffer(addid) diff --git a/src/axom/sidre/tests/sidre_external_Py.py b/src/axom/sidre/tests/sidre_external_Py.py index d08c1145b2..586b9f93e7 100644 --- a/src/axom/sidre/tests/sidre_external_Py.py +++ b/src/axom/sidre/tests/sidre_external_Py.py @@ -4,17 +4,17 @@ # # SPDX-License-Identifier: (BSD-3-Clause) -import pysidre +import axom.sidre as sidre import numpy as np from conduit import Node -############################################################################### +# ------------------------------------------------------------------------------ # Tests from sidre_external.cpp -############################################################################### +# ------------------------------------------------------------------------------ def test_create_external_view(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() length = 11 @@ -29,26 +29,25 @@ def test_create_external_view(): view = None match i: case 0: - view = root.createView("data0", pysidre.TypeID.INT64_ID, length, idata) + view = root.createView("data0", sidre.TypeID.INT64_ID, length, idata) case 1: - view = root.createView("data1", pysidre.TypeID.INT64_ID, + view = root.createView("data1", sidre.TypeID.INT64_ID, length).setExternalData(idata) case 2: - view = root.createView("data2").setExternalData(pysidre.TypeID.INT64_ID, length, + view = root.createView("data2").setExternalData(sidre.TypeID.INT64_ID, length, idata) case 3: - view = root.createView("data3", idata).apply(pysidre.TypeID.INT64_ID, length) + view = root.createView("data3", idata).apply(sidre.TypeID.INT64_ID, length) case 4: - view = root.createViewWithShape("data4", pysidre.TypeID.INT64_ID, ndims, shape, - idata) + view = root.createViewWithShape("data4", sidre.TypeID.INT64_ID, ndims, shape, idata) case 5: - view = root.createViewWithShape("data5", pysidre.TypeID.INT64_ID, ndims, + view = root.createViewWithShape("data5", sidre.TypeID.INT64_ID, ndims, shape).setExternalData(idata) case 6: - view = root.createView("data6").setExternalData(pysidre.TypeID.INT64_ID, ndims, - shape, idata) + view = root.createView("data6").setExternalData(sidre.TypeID.INT64_ID, ndims, shape, + idata) case 7: - view = root.createView("data7", idata).apply(pysidre.TypeID.INT64_ID, ndims, shape) + view = root.createView("data7", idata).apply(sidre.TypeID.INT64_ID, ndims, shape) assert view is not None assert root.getNumViews() == i + 1 @@ -60,7 +59,7 @@ def test_create_external_view(): assert view.isExternal() assert not view.isOpaque() - assert view.getTypeID() == pysidre.TypeID.INT64_ID + assert view.getTypeID() == sidre.TypeID.INT64_ID assert view.getNumElements() == length view.print() @@ -73,7 +72,7 @@ def test_create_external_view(): def test_verify_external_layout(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() SZ = 11 @@ -84,7 +83,7 @@ def test_verify_external_layout(): associated with external pointers (described or undescribed).""") # Create some internal views - root.createViewAndAllocate("int/desc/bufferview", pysidre.TypeID.INT64_ID, SZ) + root.createViewAndAllocate("int/desc/bufferview", sidre.TypeID.INT64_ID, SZ) root.createViewScalar("int/scalar/scalarview", SZ) root.createViewString("int/string/stringview", "A string") @@ -106,7 +105,7 @@ def test_verify_external_layout(): assert emptyNode.number_of_children() == 0 # Create some external views - root.createView("ext/desc/external_desc", pysidre.TypeID.INT64_ID, SZ, extData) + root.createView("ext/desc/external_desc", sidre.TypeID.INT64_ID, SZ, extData) root.createView("ext/undesc/external_opaque").setExternalData(extData) # Sanity check on the external views @@ -149,11 +148,11 @@ def test_verify_external_layout(): def test_save_load_external_view(): - if not pysidre.AXOM_USE_HDF5: - print("pysidre.Group.loadExternalData() is only implemented for the 'sidre_hdf5' protocol") + if not sidre.AXOM_USE_HDF5: + print("sidre.Group.loadExternalData() is only implemented for the 'sidre_hdf5' protocol") return - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() length = 11 @@ -162,13 +161,13 @@ def test_save_load_external_view(): ddata = np.array([ii * 2.0 for ii in range(length)], dtype=np.float64) # Create views with external data - root.createView("idata", idata).apply(pysidre.TypeID.INT64_ID, length) - root.createView("ddata", ddata).apply(pysidre.TypeID.FLOAT64_ID, length) + root.createView("idata", idata).apply(sidre.TypeID.INT64_ID, length) + root.createView("ddata", ddata).apply(sidre.TypeID.FLOAT64_ID, length) assert root.getNumViews() == 2 root.save("sidre_external_save_load_external_view", "sidre_hdf5") - ds2 = pysidre.DataStore() + ds2 = sidre.DataStore() load_group = ds2.getRoot() # Load from file, the Views with external data will be described but @@ -184,8 +183,8 @@ def test_save_load_external_view(): assert load_ddata.isExternal() assert load_idata.getNumElements() == length assert load_ddata.getNumElements() == length - assert load_idata.getTypeID() == pysidre.TypeID.INT64_ID - assert load_ddata.getTypeID() == pysidre.TypeID.FLOAT64_ID + assert load_idata.getTypeID() == sidre.TypeID.INT64_ID + assert load_ddata.getTypeID() == sidre.TypeID.FLOAT64_ID # Create arrays that will serve as locations for external data new_idata = np.zeros(length, dtype=np.int64) @@ -215,25 +214,25 @@ def test_save_load_external_view(): assert ddata_chk[ii] == ddata[ii] -############################################################################### +# ------------------------------------------------------------------------------ # Tests from sidre_external_F.f -############################################################################### +# ------------------------------------------------------------------------------ # External numpy array via python # Register with datastore then # Query metadata using datastore API. def test_external_int(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() iarray = np.array(range(1, 11)) view = root.createView("iarray", iarray) - view.apply(pysidre.TypeID.INT64_ID, 10) + view.apply(sidre.TypeID.INT64_ID, 10) assert view.isExternal() == True - assert view.getTypeID() == pysidre.TypeID.INT64_ID + assert view.getTypeID() == sidre.TypeID.INT64_ID assert view.getNumElements() == np.size(iarray) assert view.getNumDimensions() == 1 @@ -247,7 +246,7 @@ def test_external_int(): def test_external_int_3d(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() # create 3D numpy array @@ -258,10 +257,10 @@ def test_external_int_3d(): for k in range(4): iarray[i, j, k] = (i + 1) * 100 + (j + 1) * 10 + (k + 1) view = root.createView("iarray", iarray) - view.apply(pysidre.TypeID.INT64_ID, 3, np.array([2, 3, 4])) + view.apply(sidre.TypeID.INT64_ID, 3, np.array([2, 3, 4])) assert view.isExternal() == True - assert view.getTypeID() == pysidre.TypeID.INT64_ID + assert view.getTypeID() == sidre.TypeID.INT64_ID assert view.getNumElements() == np.size(iarray) assert view.getNumDimensions() == 3 @@ -278,14 +277,14 @@ def test_external_int_3d(): # check other types def test_external_float(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() darray = np.array([(i + 0.5) for i in range(1, 11)]) view = root.createView("darray", darray) - view.apply(pysidre.TypeID.FLOAT64_ID, 10) + view.apply(sidre.TypeID.FLOAT64_ID, 10) - assert view.getTypeID() == pysidre.TypeID.FLOAT64_ID + assert view.getTypeID() == sidre.TypeID.FLOAT64_ID assert view.getNumElements() == np.size(darray) dpointer = view.getDataArray() @@ -294,15 +293,15 @@ def test_external_float(): # Datastore owns a multi-dimension array. def test_datastore_int_3d(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() extents_in = [2, 3, 4] - view = root.createViewWithShapeAndAllocate("iarray", pysidre.TypeID.INT32_ID, 3, extents_in) + view = root.createViewWithShapeAndAllocate("iarray", sidre.TypeID.INT32_ID, 3, extents_in) ipointer = view.getDataArray() - assert view.getTypeID() == pysidre.TypeID.INT32_ID + assert view.getTypeID() == sidre.TypeID.INT32_ID assert view.getNumElements() == np.size(ipointer) assert view.getNumDimensions() == 3 assert view.getNumDimensions() == ipointer.ndim @@ -316,9 +315,9 @@ def test_datastore_int_3d(): # Reshape as 1D using shape extents_in[0] = np.size(ipointer) - view.apply(pysidre.TypeID.INT32_ID, 1, np.array([extents_in[0]])) + view.apply(sidre.TypeID.INT32_ID, 1, np.array([extents_in[0]])) assert view.getNumElements() == np.size(ipointer) # Reshape as 1D using length - view.apply(pysidre.TypeID.INT32_ID, extents_in[0]) + view.apply(sidre.TypeID.INT32_ID, extents_in[0]) assert view.getNumElements() == np.size(ipointer) diff --git a/src/axom/sidre/tests/sidre_group_Py.py b/src/axom/sidre/tests/sidre_group_Py.py index 9fbf58335d..70a28831df 100644 --- a/src/axom/sidre/tests/sidre_group_Py.py +++ b/src/axom/sidre/tests/sidre_group_Py.py @@ -4,11 +4,11 @@ # # SPDX-License-Identifier: (BSD-3-Clause) -import pysidre +import axom.sidre as sidre import numpy as np from conduit import Node -if pysidre.AXOM_USE_HDF5: +if sidre.AXOM_USE_HDF5: NPROTOCOLS = 3 PROTOCOLS = ["sidre_json", "sidre_hdf5", "json"] else: @@ -20,7 +20,7 @@ # getName() # ------------------------------------------------------------------------------ def test_get_name(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() grp = root.createGroup("test") @@ -34,7 +34,7 @@ def test_get_name(): # getPath(), getPathName() # ------------------------------------------------------------------------------ def test_get_path_name(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() group = root.createGroup("test/a/b/c") grp2 = root.getGroup("test/a") @@ -61,7 +61,7 @@ def test_get_path_name(): # createGroup(), getGroup(), hasGroup() with path strings #------------------------------------------------------------------------------ def test_group_with_path(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() # Test full path access when building incrementally @@ -111,7 +111,7 @@ def test_group_with_path(): assert root.hasGroup(1) assert root.hasGroup(2) assert not root.hasGroup(3) - assert not root.hasGroup(pysidre.InvalidIndex) + assert not root.hasGroup(sidre.InvalidIndex) testbnumgroups = group_testa.getGroup("testb").getNumGroups() group_cdup = group_testa.createGroup("testb/testc") @@ -124,7 +124,7 @@ def test_group_with_path(): # createGroup(), destroyGroup() with path strings #------------------------------------------------------------------------------ def test_destroy_group_with_path(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() # Test full path access when building incrementally @@ -155,7 +155,7 @@ def test_destroy_group_with_path(): # Verify getParent() # ------------------------------------------------------------------------------ def test_get_parent(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() parent = root.createGroup("parent") child = parent.createGroup("child") @@ -167,7 +167,7 @@ def test_get_parent(): # Verify getDataStore() # ------------------------------------------------------------------------------ def test_get_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() grp = root.createGroup("parent") @@ -181,7 +181,7 @@ def test_get_datastore(): # Verify getGroup() # ------------------------------------------------------------------------------ def test_get_group(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() parent = root.createGroup("parent") @@ -203,7 +203,7 @@ def test_get_group(): # getView() # ------------------------------------------------------------------------------ def test_get_view(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() parent = root.createGroup("parent") @@ -220,10 +220,10 @@ def test_get_view(): def test_group_and_view_checksum(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() group = root.createGroup("checksum_group") - view = group.createViewAndAllocate("values", pysidre.TypeID.INT32_ID, 4) + view = group.createViewAndAllocate("values", sidre.TypeID.INT32_ID, 4) data = view.getDataArray() data[:] = np.array([1, 2, 3, 4], dtype=np.int32) @@ -270,7 +270,7 @@ def test_group_and_view_checksum(): # createView, hasView(), getView(), destroyView() with path strings #------------------------------------------------------------------------------ def test_view_with_path(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() # Test with full path access when building incrementally @@ -345,7 +345,7 @@ def test_view_with_path(): # Verify getViewName() and getViewIndex() #------------------------------------------------------------------------------ def test_get_view_name_index(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() parent = root.createGroup("parent") @@ -367,18 +367,18 @@ def test_get_view_name_index(): assert view2.getName() == name2 idx3 = parent.getViewIndex("view3") - assert idx3 == pysidre.InvalidIndex + assert idx3 == sidre.InvalidIndex name3 = parent.getViewName(idx3) assert name3 == "" - assert not pysidre.nameIsValid(name3) + assert not sidre.nameIsValid(name3) #------------------------------------------------------------------------------ # Verify getFirstValidGroupIndex() and getNextValidGroupIndex() #------------------------------------------------------------------------------ def test_get_first_and_next_group_index(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() parent = root.createGroup("parent") @@ -392,7 +392,7 @@ def test_get_first_and_next_group_index(): assert idx1 == 0 assert idx2 == 1 - assert idx3 == pysidre.InvalidIndex + assert idx3 == sidre.InvalidIndex group1out = parent.getGroup(idx1) group2out = parent.getGroup(idx2) @@ -405,15 +405,15 @@ def test_get_first_and_next_group_index(): badidx1 = emptygrp.getFirstValidGroupIndex() badidx2 = emptygrp.getNextValidGroupIndex(badidx1) - assert badidx1 == pysidre.InvalidIndex - assert badidx2 == pysidre.InvalidIndex + assert badidx1 == sidre.InvalidIndex + assert badidx2 == sidre.InvalidIndex #------------------------------------------------------------------------------ # Verify Groups holding items in the list format #------------------------------------------------------------------------------ def test_child_lists(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() # parent is a Group in list format. @@ -433,8 +433,8 @@ def test_child_lists(): else: unnamed_view = parent.createViewString("", "foo") if not unnamed_view.isApplied(): - unnamed_view.apply(pysidre.TypeID.INT_ID, i) - unnamed_view.allocate(pysidre.TypeID.INT_ID, i) + unnamed_view.apply(sidre.TypeID.INT_ID, i) + unnamed_view.allocate(sidre.TypeID.INT_ID, i) vdata = unnamed_view.getDataArray() # Returns numpy array for j in range(i): vdata[j] = j + 3 @@ -452,7 +452,7 @@ def test_child_lists(): # Access data from unnamed Groups held by parent. scalars = set() idx = parent.getFirstValidGroupIndex() - while pysidre.indexIsValid(idx): + while sidre.indexIsValid(idx): unnamed_group = parent.getGroup(idx) val_view = unnamed_group.getView("val") val = val_view.getDataInt() @@ -467,7 +467,7 @@ def test_child_lists(): # Destroy five of the unnamed Groups held by parent. idx = parent.getFirstValidGroupIndex() - while pysidre.indexIsValid(idx): + while sidre.indexIsValid(idx): if idx % 2 == 1: parent.destroyGroup(idx) idx = parent.getNextValidGroupIndex(idx) @@ -478,10 +478,10 @@ def test_child_lists(): # Access data from the unnamed Views. idx = parent.getFirstValidViewIndex() - while pysidre.indexIsValid(idx): + while sidre.indexIsValid(idx): unnamed_view = parent.getView(idx) if idx % 3 == 0: - assert unnamed_view.getTypeID() == pysidre.TypeID.INT32_ID + assert unnamed_view.getTypeID() == sidre.TypeID.INT32_ID num_elems = unnamed_view.getNumElements() assert num_elems == idx vdata = unnamed_view.getDataArray() @@ -504,7 +504,7 @@ def test_child_lists(): # Verify results with various path arguments for items in list #------------------------------------------------------------------------------ def test_list_item_names(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() # Create a group that uses the list format. @@ -572,7 +572,7 @@ def test_list_item_names(): #------------------------------------------------------------------------------ def test_string_list(): # Round-trip test from Python list of strings to Group and back. - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() str_vec = [ @@ -596,7 +596,7 @@ def test_string_list(): # Get strings from the Group. idx = my_strings.getFirstValidViewIndex() - while pysidre.indexIsValid(idx): + while sidre.indexIsValid(idx): str_view = my_strings.getView(idx) assert str_view is not None assert str_view.isString() @@ -611,7 +611,7 @@ def test_string_list(): # Iterate Groups with getFirstValidGroupIndex, getNextValidGroupIndex #------------------------------------------------------------------------------ def test_iterate_groups(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() parent = root.createGroup("parent") @@ -631,7 +631,7 @@ def test_iterate_groups(): groupcount = 0 idx = parent.getFirstValidGroupIndex() - while pysidre.indexIsValid(idx): + while sidre.indexIsValid(idx): groupcount += 1 idx = parent.getNextValidGroupIndex(idx) assert groupcount == 4 @@ -642,7 +642,7 @@ def test_iterate_groups(): #------------------------------------------------------------------------------ def test_iterate_groups_with_iterator(): - ds = pysidre.DataStore() + ds = sidre.DataStore() foo_group = ds.getRoot().createGroup("foo") foo_group.createGroup("bar_group") foo_group.createGroup("bar_group/child_1") @@ -684,7 +684,7 @@ def test_iterate_groups_with_iterator(): # Verify getFirstValidViewIndex() and getNextValidIndex() #------------------------------------------------------------------------------ def test_get_first_and_next_view_index(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() parent = root.createGroup("parent") @@ -698,7 +698,7 @@ def test_get_first_and_next_view_index(): idx3 = parent.getNextValidViewIndex(idx2) assert idx1 == 0 assert idx2 == 1 - assert idx3 == pysidre.InvalidIndex + assert idx3 == sidre.InvalidIndex view1out = parent.getView(idx1) view2out = parent.getView(idx2) @@ -710,15 +710,15 @@ def test_get_first_and_next_view_index(): badidx1 = emptygrp.getFirstValidViewIndex() badidx2 = emptygrp.getNextValidViewIndex(badidx1) - assert badidx1 == pysidre.InvalidIndex - assert badidx2 == pysidre.InvalidIndex + assert badidx1 == sidre.InvalidIndex + assert badidx2 == sidre.InvalidIndex #------------------------------------------------------------------------------ # Iterate Views with getFirstValidViewIndex, getNextValidViewIndex #------------------------------------------------------------------------------ def test_iterate_views(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() parent = root.createGroup("parent") @@ -738,7 +738,7 @@ def test_iterate_views(): viewcount = 0 idx = parent.getFirstValidViewIndex() - while pysidre.indexIsValid(idx): + while sidre.indexIsValid(idx): viewcount += 1 idx = parent.getNextValidViewIndex(idx) assert viewcount == 9 @@ -748,7 +748,7 @@ def test_iterate_views(): # Verify getGroupName() and getGroupIndex() #------------------------------------------------------------------------------ def test_get_group_name_index(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() parent = root.createGroup("parent") @@ -769,11 +769,11 @@ def test_get_group_name_index(): assert grp2.getName() == name2 idx3 = parent.getGroupIndex("grp3") - assert idx3 == pysidre.InvalidIndex + assert idx3 == sidre.InvalidIndex name3 = parent.getGroupName(idx3) assert name3 == "" - assert not pysidre.nameIsValid(name3) + assert not sidre.nameIsValid(name3) # ------------------------------------------------------------------------------ @@ -784,7 +784,7 @@ def test_get_group_name_index(): # hasView() # ------------------------------------------------------------------------------ def test_create_destroy_has_view(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() group = root.createGroup("parent") @@ -817,22 +817,21 @@ def test_create_destroy_has_view(): assert not group.hasView("view") # Try API call that specifies specific type and length - group.createViewAndAllocate("viewWithLength1", pysidre.TypeID.INT32_ID, 50) + group.createViewAndAllocate("viewWithLength1", sidre.TypeID.INT32_ID, 50) iview2 = group.getViewIndex("viewWithLength1") assert iview == iview2 # reuse slot # Error condition check - try again with duplicate name, should be a no-op - assert group.createViewAndAllocate("viewWithLength1", pysidre.TypeID.FLOAT64_ID, 50) is None + assert group.createViewAndAllocate("viewWithLength1", sidre.TypeID.FLOAT64_ID, 50) is None group.destroyViewAndData("viewWithLength1") assert not group.hasView("viewWithLength1") # Should not allow negative length - assert group.createViewAndAllocate("viewWithLengthBadLen", pysidre.TypeID.FLOAT64_ID, - -1) is None + assert group.createViewAndAllocate("viewWithLengthBadLen", sidre.TypeID.FLOAT64_ID, -1) is None # Try API call that specifies data type in another way - group.createViewAndAllocate("viewWithLength2", pysidre.TypeID.FLOAT64_ID, 50) - assert group.createViewAndAllocate("viewWithLength2", pysidre.TypeID.FLOAT64_ID, 50) is None + group.createViewAndAllocate("viewWithLength2", sidre.TypeID.FLOAT64_ID, 50) + assert group.createViewAndAllocate("viewWithLength2", sidre.TypeID.FLOAT64_ID, 50) is None # Destroy view and its buffer using index indx = group.getFirstValidViewIndex() @@ -843,7 +842,7 @@ def test_create_destroy_has_view(): assert ds.getBuffer(bindx) is None # Destroy view but not the buffer - view = group.createViewAndAllocate("viewWithLength2", pysidre.TypeID.INT_ID, 50) + view = group.createViewAndAllocate("viewWithLength2", sidre.TypeID.INT_ID, 50) buff = view.getBuffer() group.destroyView("viewWithLength2") assert buff.isAllocated() @@ -853,10 +852,10 @@ def test_create_destroy_has_view(): # createViewAndAllocate() with zero-sized array #------------------------------------------------------------------------------ def test_create_zero_sized_view(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() - zero_sized_view = root.createViewAndAllocate("foo", pysidre.TypeID.INT_ID, 0) + zero_sized_view = root.createViewAndAllocate("foo", sidre.TypeID.INT_ID, 0) assert zero_sized_view.isDescribed() assert zero_sized_view.isAllocated() @@ -865,7 +864,7 @@ def test_create_zero_sized_view(): # Verify createGroup(), destroyGroup(), hasGroup() #------------------------------------------------------------------------------ def test_create_destroy_has_group(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() grp = root.createGroup("grp") @@ -884,7 +883,7 @@ def test_create_destroy_has_group(): # Test various destroy methods #------------------------------------------------------------------------------ def test_destroy_group_and_data(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() group0 = root.createGroup("group0") group1 = root.createGroup("group1") @@ -894,12 +893,12 @@ def test_destroy_group_and_data(): child3 = group1.createGroup("child3") child4 = group1.createGroup("child4") - child0.createViewAndAllocate("intview", pysidre.TypeID.INT_ID, 15) + child0.createViewAndAllocate("intview", sidre.TypeID.INT_ID, 15) foo0 = child0.createGroup("foo") child0.createGroup("empty") child0.createViewScalar("sclview", 3.14159) child0.createViewString("strview", "Hello world.") - foo0.createViewAndAllocate("fooview", pysidre.TypeID.FLOAT64_ID, 12) + foo0.createViewAndAllocate("fooview", sidre.TypeID.FLOAT64_ID, 12) int0_view = child0.getView("intview") int0_vals = int0_view.getDataArray() @@ -919,33 +918,33 @@ def test_destroy_group_and_data(): flt_idx = fltbuf.getIndex() # Attach buffers to views in other children - child1.createView("intview", pysidre.TypeID.INT_ID, 15, intbuf) + child1.createView("intview", sidre.TypeID.INT_ID, 15, intbuf) foo1 = child1.createGroup("foo") child1.createGroup("empty") child1.createViewScalar("sclview", 3.14159) child1.createViewString("strview", "Hello world.") - foo1.createView("fooview", pysidre.TypeID.FLOAT64_ID, 12, fltbuf) + foo1.createView("fooview", sidre.TypeID.FLOAT64_ID, 12, fltbuf) - child2.createView("intview", pysidre.TypeID.INT_ID, 15, intbuf) + child2.createView("intview", sidre.TypeID.INT_ID, 15, intbuf) foo2 = child2.createGroup("foo") child2.createGroup("empty") child2.createViewScalar("sclview", 3.14159) child2.createViewString("strview", "Hello world.") - foo2.createView("fooview", pysidre.TypeID.FLOAT64_ID, 12, fltbuf) + foo2.createView("fooview", sidre.TypeID.FLOAT64_ID, 12, fltbuf) - child3.createView("intview", pysidre.TypeID.INT_ID, 15, intbuf) + child3.createView("intview", sidre.TypeID.INT_ID, 15, intbuf) foo3 = child3.createGroup("foo") child3.createGroup("empty") child3.createViewScalar("sclview", 3.14159) child3.createViewString("strview", "Hello world.") - foo3.createView("fooview", pysidre.TypeID.FLOAT64_ID, 12, fltbuf) + foo3.createView("fooview", sidre.TypeID.FLOAT64_ID, 12, fltbuf) - child4.createView("intview", pysidre.TypeID.INT_ID, 15, intbuf) + child4.createView("intview", sidre.TypeID.INT_ID, 15, intbuf) foo4 = child4.createGroup("foo") child4.createGroup("empty") child4.createViewScalar("sclview", 3.14159) child4.createViewString("strview", "Hello world.") - foo4.createView("fooview", pysidre.TypeID.FLOAT64_ID, 12, fltbuf) + foo4.createView("fooview", sidre.TypeID.FLOAT64_ID, 12, fltbuf) # Beginning state: 2 Buffers, each attached to 5 Views. assert ds.getNumBuffers() == 2 @@ -1029,7 +1028,7 @@ def test_destroy_group_and_data(): def test_group_name_collisions(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() flds = root.createGroup("fields") flds.createView("a") @@ -1055,13 +1054,13 @@ def test_group_name_collisions(): # Print all group names idx = root.getFirstValidGroupIndex() - while pysidre.indexIsValid(idx): + while sidre.indexIsValid(idx): print(root.getGroup(idx).getName()) idx = root.getNextValidGroupIndex(idx) def test_view_copy_move(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() flds = root.createGroup("fields") extdata = np.array([0] * 10) @@ -1071,9 +1070,9 @@ def test_view_copy_move(): # Create views in different states views[0] = flds.createView("empty0") - views[1] = flds.createView("empty1", pysidre.TypeID.INT32_ID, 10) - views[2] = flds.createViewAndAllocate("buffer", pysidre.TypeID.INT32_ID, 10) - views[3] = flds.createView("external", pysidre.TypeID.INT32_ID, 10) + views[1] = flds.createView("empty1", sidre.TypeID.INT32_ID, 10) + views[2] = flds.createViewAndAllocate("buffer", sidre.TypeID.INT32_ID, 10) + views[3] = flds.createView("external", sidre.TypeID.INT32_ID, 10) views[3].setExternalData(extdata) views[4] = flds.createViewScalar("scalar", 25) views[5] = flds.createViewString("string", "I am string") @@ -1163,7 +1162,7 @@ def test_view_copy_move(): def test_groups_move_copy(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() flds = root.createGroup("fields") @@ -1254,7 +1253,7 @@ def test_groups_move_copy(): def test_group_deep_copy(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() flds = root.createGroup("fields") @@ -1274,14 +1273,14 @@ def test_group_deep_copy(): assert flds.hasGroup("b") viewlen = 8 - ownsbuf = ga.createViewAndAllocate("ownsbuf", pysidre.TypeID.INT32_ID, viewlen) + ownsbuf = ga.createViewAndAllocate("ownsbuf", sidre.TypeID.INT32_ID, viewlen) int_vals = ownsbuf.getDataArray() for i in range(viewlen): int_vals[i] = i + 1 buflen = 24 dbuff = ds.createBuffer() - dbuff.allocate(pysidre.TypeID.FLOAT64_ID, buflen) + dbuff.allocate(sidre.TypeID.FLOAT64_ID, buflen) buf_ptr = dbuff.getDataArray() for i in range(buflen): buf_ptr[i] = 2.0 * float(i) @@ -1299,7 +1298,7 @@ def test_group_deep_copy(): ext_array = np.array([-1.0 * float(i) for i in range(extlen)]) for i in range(NUM_VIEWS): - gb.createView(names[i], ext_array).apply(pysidre.TypeID.FLOAT64_ID, size[i], offset[i], + gb.createView(names[i], ext_array).apply(sidre.TypeID.FLOAT64_ID, size[i], offset[i], stride[i]) deep_copy = root.createGroup("deep_copy") @@ -1351,15 +1350,15 @@ def test_group_deep_copy(): def test_create_destroy_view_and_buffer2(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() grp = root.createGroup("grp") viewName1 = "viewBuffer1" viewName2 = "viewBuffer2" - view1 = grp.createViewAndAllocate(viewName1, pysidre.TypeID.INT_ID, 1) - view2 = grp.createViewAndAllocate(viewName2, pysidre.TypeID.INT_ID, 1) + view1 = grp.createViewAndAllocate(viewName1, sidre.TypeID.INT_ID, 1) + view2 = grp.createViewAndAllocate(viewName2, sidre.TypeID.INT_ID, 1) assert grp.hasView(viewName1) assert grp.getView(viewName1) == view1 @@ -1384,7 +1383,7 @@ def test_create_destroy_view_and_buffer2(): def test_create_destroy_alloc_view_and_buffer(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() grp = root.createGroup("grp") @@ -1393,7 +1392,7 @@ def test_create_destroy_alloc_view_and_buffer(): # Use create + alloc convenience methods # This one is the DataType method - view1 = grp.createViewAndAllocate(viewName1, pysidre.TypeID.INT_ID, 10) + view1 = grp.createViewAndAllocate(viewName1, sidre.TypeID.INT_ID, 10) assert grp.hasChildView(viewName1) assert grp.getView(viewName1) == view1 @@ -1409,11 +1408,11 @@ def test_create_destroy_alloc_view_and_buffer(): def test_create_view_of_buffer_with_schema(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() # Use create + alloc convenience methods - base = root.createViewAndAllocate("base", pysidre.TypeID.INT_ID, 10) + base = root.createViewAndAllocate("base", sidre.TypeID.INT_ID, 10) base_vals = base.getDataArray() for i in range(10): if i < 5: @@ -1426,7 +1425,7 @@ def test_create_view_of_buffer_with_schema(): # Create two views into this buffer # View for the first 5 values sub_a = root.createView("sub_a", base_buff) - sub_a.apply(pysidre.TypeID.INT_ID, 5) + sub_a.apply(sidre.TypeID.INT_ID, 5) sub_a_vals = sub_a.getDataArray() for i in range(5): @@ -1434,15 +1433,15 @@ def test_create_view_of_buffer_with_schema(): def test_create_destroy_view_and_data(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() grp = root.createGroup("grp") view_name1 = "viewBuffer1" view_name2 = "viewBuffer2" - view1 = grp.createViewAndAllocate(view_name1, pysidre.TypeID.INT32_ID, 1) - view2 = grp.createViewAndAllocate(view_name2, pysidre.TypeID.INT32_ID, 1) + view1 = grp.createViewAndAllocate(view_name1, sidre.TypeID.INT32_ID, 1) + view2 = grp.createViewAndAllocate(view_name2, sidre.TypeID.INT32_ID, 1) assert grp.hasView(view_name1) assert grp.getView(view_name1) == view1 @@ -1460,7 +1459,7 @@ def test_create_destroy_view_and_data(): def test_create_destroy_alloc_view_and_data(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() grp = root.createGroup("grp") @@ -1469,7 +1468,7 @@ def test_create_destroy_alloc_view_and_data(): # Use create + alloc convenience methods # this one is the DataType & method - view1 = grp.createViewAndAllocate(view_name1, pysidre.TypeID.INT32_ID, 10) + view1 = grp.createViewAndAllocate(view_name1, sidre.TypeID.INT32_ID, 10) assert grp.hasView(view_name1) assert grp.getView(view_name1) == view1 @@ -1484,12 +1483,12 @@ def test_create_destroy_alloc_view_and_data(): def test_create_view_of_buffer_with_datatype(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() # Use create + alloc convenience methods # this one is the DataType & method - base = root.createViewAndAllocate("base", pysidre.TypeID.INT32_ID, 10) + base = root.createViewAndAllocate("base", sidre.TypeID.INT32_ID, 10) base_vals = base.getDataArray() base_vals[0:5] = 10 @@ -1498,7 +1497,7 @@ def test_create_view_of_buffer_with_datatype(): base_buff = base.getBuffer() # Create view into this buffer - sub_a = root.createView("sub_a", pysidre.TypeID.INT32_ID, 10, base_buff) + sub_a = root.createView("sub_a", sidre.TypeID.INT32_ID, 10, base_buff) sub_a_vals = root.getView("sub_a").getDataArray() @@ -1510,7 +1509,7 @@ def test_create_view_of_buffer_with_datatype(): def test_save_restore_empty_datastore(): file_path_base = "py_sidre_empty_datastore_" - ds1 = pysidre.DataStore() + ds1 = sidre.DataStore() root1 = ds1.getRoot() for i in range(NPROTOCOLS): @@ -1522,7 +1521,7 @@ def test_save_restore_empty_datastore(): continue file_path = file_path_base + PROTOCOLS[i] - ds2 = pysidre.DataStore() + ds2 = sidre.DataStore() root2 = ds2.getRoot() root2.load(file_path, PROTOCOLS[i]) @@ -1534,7 +1533,7 @@ def test_save_restore_empty_datastore(): def test_save_restore_scalars_and_strings(): file_path_base = "py_sidre_save_scalars_and_strings_" - ds1 = pysidre.DataStore() + ds1 = sidre.DataStore() root1 = ds1.getRoot() view = root1.createViewScalar("i0", 1) @@ -1551,7 +1550,7 @@ def test_save_restore_scalars_and_strings(): continue file_path = file_path_base + PROTOCOLS[i] - ds2 = pysidre.DataStore() + ds2 = sidre.DataStore() root2 = ds2.getRoot() root2.load(file_path, PROTOCOLS[i]) @@ -1590,13 +1589,13 @@ def test_save_restore_external_data(): int2d1 = np.column_stack((foo1, foo1 + nfoo)) int2d2 = np.zeros((10, 2), dtype=int) - ds1 = pysidre.DataStore() + ds1 = sidre.DataStore() root1 = ds1.getRoot() - root1.createView("external_array", pysidre.TypeID.INT64_ID, nfoo, foo1) - root1.createView("empty_array", pysidre.TypeID.INT64_ID, 0, foo3) + root1.createView("external_array", sidre.TypeID.INT64_ID, nfoo, foo1) + root1.createView("empty_array", sidre.TypeID.INT64_ID, 0, foo3) root1.createView("external_undescribed").setExternalData(foo4) - root1.createViewWithShape("int2d", pysidre.TypeID.INT64_ID, 2, shape, int2d1) + root1.createViewWithShape("int2d", sidre.TypeID.INT64_ID, 2, shape, int2d1) for protocol in PROTOCOLS: file_path = file_path_base + protocol @@ -1609,7 +1608,7 @@ def test_save_restore_external_data(): continue file_path = file_path_base + protocol - ds2 = pysidre.DataStore() + ds2 = sidre.DataStore() root2 = ds2.getRoot() assert root2.load(file_path, protocol) == True @@ -1619,14 +1618,14 @@ def test_save_restore_external_data(): view1 = root2.getView("external_array") assert view1.isExternal() == True, "external_array is external" assert view1.isDescribed() == True, "external_array is described" - assert view1.getTypeID() == pysidre.TypeID.INT64_ID, "external_array get TypeId" + assert view1.getTypeID() == sidre.TypeID.INT64_ID, "external_array get TypeId" assert view1.getNumElements() == nfoo, "external_array get num elements" view1.setExternalData(foo2) view2 = root2.getView("empty_array") assert view2.isExternal() == True, "empty_array is external" assert view2.isDescribed() == True, "empty_array is described" - assert view2.getTypeID() == pysidre.TypeID.INT64_ID, "empty_array get TypeId" + assert view2.getTypeID() == sidre.TypeID.INT64_ID, "empty_array get TypeId" view2.setExternalData(foo3) view3 = root2.getView("external_undescribed") @@ -1637,7 +1636,7 @@ def test_save_restore_external_data(): view4 = root2.getView("int2d") assert view4.isExternal() == True, "int2d is external" assert view4.isDescribed() == True, "int2d is described" - assert view4.getTypeID() == pysidre.TypeID.INT64_ID, "int2d get TypeId" + assert view4.getTypeID() == sidre.TypeID.INT64_ID, "int2d get TypeId" assert view4.getNumElements() == nfoo * 2, "int2d get num elements" assert view4.getNumDimensions() == 2, "int2d get num dimensions" @@ -1664,14 +1663,14 @@ def test_save_restore_other(): file_path_base = "py_sidre_empty_other_" ndata = 10 - ds1 = pysidre.DataStore() + ds1 = sidre.DataStore() root1 = ds1.getRoot() shape1 = np.array([ndata, 2]) view1 = root1.createView("empty_view") - view2 = root1.createView("empty_described", pysidre.TypeID.INT32_ID, ndata) - view3 = root1.createViewWithShape("empty_shape", pysidre.TypeID.INT32_ID, 2, shape1) - view4 = root1.createViewWithShapeAndAllocate("buffer_shape", pysidre.TypeID.INT32_ID, 2, shape1) + view2 = root1.createView("empty_described", sidre.TypeID.INT32_ID, ndata) + view3 = root1.createViewWithShape("empty_shape", sidre.TypeID.INT32_ID, 2, shape1) + view4 = root1.createViewWithShapeAndAllocate("buffer_shape", sidre.TypeID.INT32_ID, 2, shape1) for protocol in PROTOCOLS: file_path = file_path_base + protocol @@ -1685,7 +1684,7 @@ def test_save_restore_other(): file_path = file_path_base + protocol - ds2 = pysidre.DataStore() + ds2 = sidre.DataStore() root2 = ds2.getRoot() root2.load(file_path, protocol) @@ -1697,13 +1696,13 @@ def test_save_restore_other(): view2 = root2.getView("empty_described") assert view2.isEmpty() == True, "empty_described is empty" assert view2.isDescribed() == True, "empty_described is described" - assert view2.getTypeID() == pysidre.TypeID.INT32_ID, "empty_described get TypeID" + assert view2.getTypeID() == sidre.TypeID.INT32_ID, "empty_described get TypeID" assert view2.getNumElements() == ndata, "empty_described get num elements" view3 = root2.getView("empty_shape") assert view3.isEmpty() == True, "empty_shape is empty" assert view3.isDescribed() == True, "empty_shape is described" - assert view3.getTypeID() == pysidre.TypeID.INT32_ID, "empty_shape get TypeID" + assert view3.getTypeID() == sidre.TypeID.INT32_ID, "empty_shape get TypeID" assert view3.getNumElements() == ndata * 2, "empty_shape get num elements" shape2 = np.zeros(7) rank, shape2 = view3.getShape(7, shape2) @@ -1713,7 +1712,7 @@ def test_save_restore_other(): view4 = root2.getView("buffer_shape") assert view4.hasBuffer() == True, "buffer_shape has buffer" assert view4.isDescribed() == True, "buffer_shape is described" - assert view4.getTypeID() == pysidre.TypeID.INT32_ID, "buffer_shape get TypeID" + assert view4.getTypeID() == sidre.TypeID.INT32_ID, "buffer_shape get TypeID" assert view4.getNumElements() == ndata * 2, "buffer_shape get num elements" shape2 = np.zeros(7) rank, shape2 = view4.getShape(7, shape2) @@ -1722,7 +1721,7 @@ def test_save_restore_other(): def test_rename_group(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() child1 = root.createGroup("g_a") child2 = root.createGroup("g_b") @@ -1748,11 +1747,11 @@ def test_rename_group(): assert child3.getName() == "g_c" # Rename root group - assert not pysidre.indexIsValid(root.getIndex()) + assert not sidre.indexIsValid(root.getIndex()) assert root.getParent() == root assert root.getName() == "" root.rename("newroot") - assert not pysidre.indexIsValid(root.getIndex()) + assert not sidre.indexIsValid(root.getIndex()) assert root.getParent() == root assert root.getName() == "newroot" @@ -1760,7 +1759,7 @@ def test_rename_group(): # Fortran comment - redo these, the C++ tests were heavily rewritten def test_save_restore_simple(): file_path = "py_out_sidre_group_save_restore_simple" - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() flds = root.createGroup("fields") @@ -1774,7 +1773,7 @@ def test_save_restore_simple(): root.save(file_path, "sidre_conduit_json") - ds2 = pysidre.DataStore() + ds2 = sidre.DataStore() root2 = ds2.getRoot() root2.load(file_path, "sidre_conduit_json") @@ -1790,7 +1789,7 @@ def test_save_restore_simple(): def test_save_restore_complex(): file_path = "py_out_sidre_group_save_restore_complex" - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() flds = root.createGroup("fields") @@ -1809,7 +1808,7 @@ def test_save_restore_complex(): root.save(file_path, "sidre_conduit_json") - ds2 = pysidre.DataStore() + ds2 = sidre.DataStore() root2 = ds2.getRoot() root2.load(file_path, "sidre_conduit_json") @@ -1839,7 +1838,7 @@ def test_save_load_preserve_contents(): file_path_base0 = "py_sidre_save_preserve_contents_tree0_" file_path_base1 = "py_sidre_save_preserve_contents_tree1_" - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() tree0 = root.createGroup("tree0") @@ -1850,7 +1849,7 @@ def test_save_load_preserve_contents(): i0_view = ga.createViewScalar("i0", 100) f0_view = ga.createViewScalar("f0", 3000.0) s0_view = gb.createViewString("s0", "foo") - i10_view = gc.createViewAndAllocate("int10", pysidre.TypeID.INT32_ID, 10) + i10_view = gc.createViewAndAllocate("int10", sidre.TypeID.INT32_ID, 10) v1_vals = i10_view.getDataArray() for i in range(10): @@ -1870,7 +1869,7 @@ def test_save_load_preserve_contents(): gy = tree1.createGroup("y") gz = tree1.createGroup("z") - i20_view = gx.createViewAndAllocate("int20", pysidre.TypeID.INT32_ID, 20) + i20_view = gx.createViewAndAllocate("int20", sidre.TypeID.INT32_ID, 20) v2_vals = i20_view.getDataArray() for i in range(20): v2_vals[i] = 2 * i @@ -1881,7 +1880,7 @@ def test_save_load_preserve_contents(): file_path1 = file_path_base1 + protocol assert tree1.save(file_path1, protocol) - dsload = pysidre.DataStore() + dsload = sidre.DataStore() ldroot = dsload.getRoot() ldtree0 = ldroot.createGroup("tree0") diff --git a/src/axom/sidre/tests/sidre_import_Py.py b/src/axom/sidre/tests/sidre_import_Py.py new file mode 100644 index 0000000000..8442f717ce --- /dev/null +++ b/src/axom/sidre/tests/sidre_import_Py.py @@ -0,0 +1,82 @@ +# Copyright (c) Lawrence Livermore National Security, LLC and other +# Axom Project Contributors. See top-level LICENSE and COPYRIGHT +# files for dates and other details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +"""Import-behavior tests for the 'axom.sidre' package. + +These check that 'axom.sidre' produces an actionable ImportError when the +compiled '_sidre' extension is absent (a component-disabled install), and that +a genuine loader failure (a discoverable '_sidre' that itself raises +ImportError) is surfaced rather than masked by the component-missing message. +""" + +import importlib +from pathlib import Path +import sys + +import pytest + + +def _clear_axom_imports(): + # These tests swap between the real staged package and synthetic packages + # under tmp_path; cached modules would otherwise bypass sys.path changes. + for name in list(sys.modules): + if name == "axom" or name.startswith("axom."): + sys.modules.pop(name, None) + + +def _sidre_init_source(): + # Exercise the staged package initializer verbatim instead of keeping a + # test-local copy of its import-error handling logic. Locate the file on + # sys.path without importing axom.sidre; re-importing the real nanobind + # extension after removing it from sys.modules can abort in some builds. + for entry in sys.path: + sidre_init = Path(entry) / "axom" / "sidre" / "__init__.py" + if sidre_init.is_file(): + return sidre_init.read_text(encoding="utf-8") + + raise RuntimeError("Could not locate axom.sidre.__init__.py on sys.path") + + +def _write_fake_axom_sidre(tmp_path, monkeypatch, sidre_init_source, sidre_extension_source=None): + # Build a minimal axom.sidre package layout. Leaving _sidre absent models a + # component-disabled install; adding _sidre.py models a discoverable module + # whose loader/import body fails. + package_root = tmp_path / "axom" + sidre_root = package_root / "sidre" + sidre_root.mkdir(parents=True) + (package_root / "__init__.py").write_text("", encoding="utf-8") + (sidre_root / "__init__.py").write_text(sidre_init_source, encoding="utf-8") + if sidre_extension_source is not None: + (sidre_root / "_sidre.py").write_text(sidre_extension_source, encoding="utf-8") + + _clear_axom_imports() + monkeypatch.syspath_prepend(str(tmp_path)) + + +def test_axom_sidre_missing_extension_gets_component_message(tmp_path, monkeypatch): + _write_fake_axom_sidre(tmp_path, monkeypatch, _sidre_init_source()) + + with pytest.raises(ImportError) as caught: + importlib.import_module("axom.sidre") + + assert "The 'axom.sidre' extension module ('_sidre') is not available" in str(caught.value) + assert "AXOM_ENABLE_SIDRE=ON" in str(caught.value) + + +def test_axom_sidre_loader_import_error_is_not_masked(tmp_path, monkeypatch): + # A discoverable _sidre that raises ImportError represents loader failures + # such as missing shared libraries; those errors must remain actionable. + _write_fake_axom_sidre( + tmp_path, + monkeypatch, + _sidre_init_source(), + "raise ImportError('libsidre_dependency_missing')\n", + ) + + with pytest.raises(ImportError) as caught: + importlib.import_module("axom.sidre") + + assert "libsidre_dependency_missing" in str(caught.value) + assert "extension module ('_sidre') is not available" not in str(caught.value) diff --git a/src/axom/sidre/tests/sidre_lifetime_Py.py b/src/axom/sidre/tests/sidre_lifetime_Py.py index b4f535fb43..2196179523 100644 --- a/src/axom/sidre/tests/sidre_lifetime_Py.py +++ b/src/axom/sidre/tests/sidre_lifetime_Py.py @@ -5,16 +5,8 @@ # SPDX-License-Identifier: (BSD-3-Clause) """Lifetime-soundness regression tests for the sidre python bindings. -Each test obtains a sidre-owned object (child proxy, ancestor proxy, harvested -iterator element, or zero-copy numpy array), drops every Python owner, forces a -garbage collection, and then *uses* the object. Before the lifetime audit these -patterns dereferenced freed memory and segfaulted; with reference_internal on -owner-chain accessors, keep_alive on iterator elements, and self-as-owner on -returned arrays, the keep_alive graph keeps the backing DataStore alive and the -accesses are safe. - -These tests therefore only "pass" against the audited bindings; against the -prior bindings they crash the interpreter (the failure mode the audit fixes). +Each test obtains a sidre-owned object (child or ancestor proxy, iterator, zero-copy numpy array), +drops every Python owner, forces a garbage collection, and then uses the object. """ import gc @@ -23,7 +15,7 @@ import numpy as np import pytest -import pysidre +import axom.sidre as sidre def _force_gc(): @@ -36,7 +28,7 @@ def _force_gc(): # Child proxies must pin their owner chain (parent -> owned child) # --------------------------------------------------------------------------- def test_root_outlives_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() del ds _force_gc() @@ -47,7 +39,7 @@ def test_root_outlives_datastore(): def test_child_group_outlives_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() grp = ds.getRoot().createGroup("a/b/c") del ds _force_gc() @@ -57,8 +49,8 @@ def test_child_group_outlives_datastore(): def test_view_outlives_datastore(): - ds = pysidre.DataStore() - view = ds.getRoot().createViewAndAllocate("field", pysidre.TypeID.INT_ID, 4) + ds = sidre.DataStore() + view = ds.getRoot().createViewAndAllocate("field", sidre.TypeID.INT_ID, 4) del ds _force_gc() assert view.getNumElements() == 4 @@ -66,8 +58,8 @@ def test_view_outlives_datastore(): def test_buffer_outlives_datastore(): - ds = pysidre.DataStore() - buff = ds.createBuffer(pysidre.TypeID.INT_ID, 8) + ds = sidre.DataStore() + buff = ds.createBuffer(sidre.TypeID.INT_ID, 8) buff.allocate() del ds _force_gc() @@ -78,7 +70,7 @@ def test_buffer_outlives_datastore(): # Ancestor proxies (child -> ancestor) must pin the object they were minted from # --------------------------------------------------------------------------- def test_owning_group_outlives_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() view = ds.getRoot().createView("v") owner = view.getOwningGroup() del ds @@ -88,7 +80,7 @@ def test_owning_group_outlives_datastore(): def test_get_datastore_back_reference(): - ds = pysidre.DataStore() + ds = sidre.DataStore() grp = ds.getRoot().createGroup("child") back = grp.getDataStore() del ds @@ -99,8 +91,8 @@ def test_get_datastore_back_reference(): def test_view_buffer_back_reference(): - ds = pysidre.DataStore() - view = ds.getRoot().createViewAndAllocate("field", pysidre.TypeID.INT_ID, 4) + ds = sidre.DataStore() + view = ds.getRoot().createViewAndAllocate("field", sidre.TypeID.INT_ID, 4) buff = view.getBuffer() del ds del view @@ -112,7 +104,7 @@ def test_view_buffer_back_reference(): # Iterator elements harvested into a list must outlive the collection + store # --------------------------------------------------------------------------- def test_harvested_views_outlive_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() for i in range(4): root.createView(f"v{i}") @@ -125,7 +117,7 @@ def test_harvested_views_outlive_datastore(): def test_harvested_groups_outlive_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() for i in range(3): root.createGroup(f"g{i}") @@ -138,9 +130,9 @@ def test_harvested_groups_outlive_datastore(): def test_harvested_buffers_outlive_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() for _ in range(3): - ds.createBuffer(pysidre.TypeID.INT_ID, 2).allocate() + ds.createBuffer(sidre.TypeID.INT_ID, 2).allocate() harvested = list(ds.buffers()) del ds _force_gc() @@ -148,7 +140,7 @@ def test_harvested_buffers_outlive_datastore(): def test_harvested_attributes_outlive_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() ds.createAttributeString("a0", "x") ds.createAttributeScalar("a1", 42) harvested = list(ds.attributes()) @@ -161,7 +153,7 @@ def test_harvested_attributes_outlive_datastore(): # Iterator adaptors must outlive the owning Group/DataStore # --------------------------------------------------------------------------- def test_views_adaptor_outlives_group_and_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() for i in range(3): root.createView(f"v{i}") @@ -173,7 +165,7 @@ def test_views_adaptor_outlives_group_and_datastore(): def test_groups_adaptor_outlives_group_and_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() for i in range(3): root.createGroup(f"g{i}") @@ -185,9 +177,9 @@ def test_groups_adaptor_outlives_group_and_datastore(): def test_buffers_adaptor_outlives_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() for _ in range(3): - ds.createBuffer(pysidre.TypeID.INT_ID, 2).allocate() + ds.createBuffer(sidre.TypeID.INT_ID, 2).allocate() adaptor = ds.buffers() del ds _force_gc() @@ -195,7 +187,7 @@ def test_buffers_adaptor_outlives_datastore(): def test_attributes_adaptor_outlives_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() ds.createAttributeString("a0", "x") ds.createAttributeScalar("a1", 42) adaptor = ds.attributes() @@ -208,7 +200,7 @@ def test_attributes_adaptor_outlives_datastore(): # Lookup accessors should return proxies that pin their owner chain # --------------------------------------------------------------------------- def test_get_view_outlives_group_and_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() root.createView("v") view = root.getView("v") @@ -219,7 +211,7 @@ def test_get_view_outlives_group_and_datastore(): def test_get_group_outlives_group_and_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() root.createGroup("g") grp = root.getGroup("g") @@ -230,8 +222,8 @@ def test_get_group_outlives_group_and_datastore(): def test_get_buffer_outlives_datastore(): - ds = pysidre.DataStore() - ds.createBuffer(pysidre.TypeID.INT_ID, 7).allocate() + ds = sidre.DataStore() + ds.createBuffer(sidre.TypeID.INT_ID, 7).allocate() buff = ds.getBuffer(0) del ds _force_gc() @@ -239,7 +231,7 @@ def test_get_buffer_outlives_datastore(): def test_get_attribute_outlives_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() ds.createAttributeString("a0", "x") attr = ds.getAttribute("a0") del ds @@ -248,7 +240,7 @@ def test_get_attribute_outlives_datastore(): def test_parent_group_outlives_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() child = ds.getRoot().createGroup("a/b") parent = child.getParent() del ds @@ -259,7 +251,7 @@ def test_parent_group_outlives_datastore(): def test_moved_group_outlives_owner_chain(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() src = root.createGroup("src") dst = root.createGroup("dst") @@ -283,8 +275,8 @@ def test_moved_group_outlives_owner_chain(): # Zero-copy numpy arrays must pin their backing View / Buffer (and DataStore) # --------------------------------------------------------------------------- def test_view_array_outlives_datastore(): - ds = pysidre.DataStore() - view = ds.getRoot().createViewAndAllocate("field", pysidre.TypeID.INT_ID, 4) + ds = sidre.DataStore() + view = ds.getRoot().createViewAndAllocate("field", sidre.TypeID.INT_ID, 4) arr = view.getDataArray() arr[:] = [10, 20, 30, 40] del ds @@ -297,8 +289,8 @@ def test_view_array_outlives_datastore(): def test_buffer_array_outlives_datastore(): - ds = pysidre.DataStore() - buff = ds.createBuffer(pysidre.TypeID.INT_ID, 4) + ds = sidre.DataStore() + buff = ds.createBuffer(sidre.TypeID.INT_ID, 4) buff.allocate() arr = buff.getDataArray() arr[:] = [1, 2, 3, 4] @@ -311,8 +303,8 @@ def test_buffer_array_outlives_datastore(): def test_view_array_survives_owner_chain_collection(): # Keep only the array; let the entire DataStore/Group/View chain be dropped. def make_array(): - ds = pysidre.DataStore() - view = ds.getRoot().createViewAndAllocate("field", pysidre.TypeID.FLOAT64_ID, 5) + ds = sidre.DataStore() + view = ds.getRoot().createViewAndAllocate("field", sidre.TypeID.FLOAT64_ID, 5) a = view.getDataArray() a[:] = np.arange(5, dtype=np.float64) return a @@ -326,13 +318,13 @@ def make_array(): # External numpy storage borrowed by Sidre must stay alive with the C++ View # --------------------------------------------------------------------------- def test_create_view_external_array_owner_survives_discarded_proxy(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() def create_external_view(): external = np.arange(6, dtype=np.int64) ref = weakref.ref(external) - root.createView("external", external).apply(pysidre.TypeID.INT64_ID, 6) + root.createView("external", external).apply(sidre.TypeID.INT64_ID, 6) return ref ref = create_external_view() @@ -342,14 +334,14 @@ def create_external_view(): def test_create_view_with_shape_external_array_owner_survives_discarded_proxy(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() shape = np.array([2, 3]) def create_external_view(): external = np.arange(6, dtype=np.int64) ref = weakref.ref(external) - root.createViewWithShape("shaped", pysidre.TypeID.INT64_ID, 2, shape, external) + root.createViewWithShape("shaped", sidre.TypeID.INT64_ID, 2, shape, external) return ref ref = create_external_view() @@ -359,14 +351,14 @@ def create_external_view(): def test_set_external_data_array_owner_survives_discarded_proxy(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() def set_external_data(): view = root.createView("external") external = np.arange(6, dtype=np.int64) ref = weakref.ref(external) - view.setExternalData(pysidre.TypeID.INT64_ID, 6, external) + view.setExternalData(sidre.TypeID.INT64_ID, 6, external) return ref ref = set_external_data() @@ -376,7 +368,7 @@ def set_external_data(): def test_set_external_data_with_shape_array_owner_survives_discarded_proxy(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() shape = np.array([2, 3]) @@ -384,7 +376,7 @@ def set_external_data(): view = root.createView("shaped") external = np.arange(6, dtype=np.int64) ref = weakref.ref(external) - view.setExternalData(pysidre.TypeID.INT64_ID, 2, shape, external) + view.setExternalData(sidre.TypeID.INT64_ID, 2, shape, external) return ref ref = set_external_data() @@ -394,11 +386,11 @@ def set_external_data(): def test_clear_releases_external_array_owner(): - ds = pysidre.DataStore() + ds = sidre.DataStore() view = ds.getRoot().createView("external") external = np.arange(6, dtype=np.int64) ref = weakref.ref(external) - view.setExternalData(pysidre.TypeID.INT64_ID, 6, external) + view.setExternalData(sidre.TypeID.INT64_ID, 6, external) del external _force_gc() assert ref() is not None @@ -408,9 +400,61 @@ def test_clear_releases_external_array_owner(): assert ref() is None +def test_set_external_data_none_clears_and_releases_pin(): + """setExternalData(None) clears the external pointer and releases the pin.""" + ds = sidre.DataStore() + view = ds.getRoot().createView("external") + + external = np.arange(6, dtype=np.int64) + ref = weakref.ref(external) + view.setExternalData(external) + + del external + _force_gc() + assert ref() is not None + assert view.isExternal() + + view.setExternalData(None) + _force_gc() + assert not view.isExternal() + assert ref() is None + + +def test_set_external_data_undescribed_array_pins(): + """The single-argument setExternalData(array) overload pins the array.""" + ds = sidre.DataStore() + view = ds.getRoot().createView("external") + + def assign(): + external = np.arange(6, dtype=np.int64) + ref = weakref.ref(external) + view.setExternalData(external) # undescribed, single-arg overload + return ref + + ref = assign() + _force_gc() + assert ref() is not None + assert view.isExternal() + + +def test_set_external_data_rejects_non_array_argument(): + """A non-array, non-None argument is rejected with a clean TypeError. + + The single-argument overload takes Optional[ndarray]; nanobind reports + 'incompatible function arguments' rather than throwing from an internal + cast, so callers get the standard overload-resolution diagnostic. + """ + ds = sidre.DataStore() + view = ds.getRoot().createView("external") + with pytest.raises(TypeError): + view.setExternalData("not an array") + with pytest.raises(TypeError): + view.setExternalData(12345) + + def test_copy_view_with_external_data_preserves_pin(): """copyView on an external View should copy the pin to prevent premature collection.""" - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() src_group = root.createGroup("src") dst_group = root.createGroup("dst") @@ -419,7 +463,7 @@ def test_copy_view_with_external_data_preserves_pin(): external = np.arange(10, dtype=np.int64) ref = weakref.ref(external) src_view = src_group.createView("original", external) - src_view.apply(pysidre.TypeID.INT64_ID, 10) + src_view.apply(sidre.TypeID.INT64_ID, 10) del external _force_gc() assert ref() is not None # Pin keeps it alive @@ -443,7 +487,7 @@ def test_copy_view_with_external_data_preserves_pin(): def test_copy_group_with_external_data_preserves_pins(): """copyGroup should recursively copy pins for all external Views in hierarchy.""" - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() src = root.createGroup("source") @@ -453,9 +497,9 @@ def test_copy_group_with_external_data_preserves_pins(): ref1 = weakref.ref(external1) ref2 = weakref.ref(external2) - src.createView("view1", external1).apply(pysidre.TypeID.INT32_ID, 5) + src.createView("view1", external1).apply(sidre.TypeID.INT32_ID, 5) child = src.createGroup("child") - child.createView("view2", external2).apply(pysidre.TypeID.INT64_ID, 8) + child.createView("view2", external2).apply(sidre.TypeID.INT64_ID, 8) del external1 del external2 @@ -490,7 +534,7 @@ def test_copy_group_with_external_data_preserves_pins(): def test_move_view_with_external_data_preserves_pin(): """moveView should preserve the pin since the View* pointer doesn't change.""" - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() src = root.createGroup("src") dst = root.createGroup("dst") @@ -499,7 +543,7 @@ def test_move_view_with_external_data_preserves_pin(): external = np.arange(12, dtype=np.int64) ref = weakref.ref(external) view = src.createView("moveable", external) - view.apply(pysidre.TypeID.INT64_ID, 12) + view.apply(sidre.TypeID.INT64_ID, 12) del external _force_gc() assert ref() is not None # Pin keeps it alive @@ -526,13 +570,13 @@ def test_move_view_with_external_data_preserves_pin(): def test_destroy_view_by_index_releases_external_pin(): """destroyView(IndexType) should release the external data pin.""" - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() external = np.arange(7, dtype=np.int32) ref = weakref.ref(external) view = root.createView("indexed", external) - view.apply(pysidre.TypeID.INT32_ID, 7) + view.apply(sidre.TypeID.INT32_ID, 7) view_idx = view.getIndex() del external del view @@ -547,16 +591,16 @@ def test_destroy_view_by_index_releases_external_pin(): def test_pin_overwrite_warning(): """Setting external data twice on the same View correctly replaces the pin.""" - ds = pysidre.DataStore() + ds = sidre.DataStore() view = ds.getRoot().createView("test") # First external data external1 = np.arange(5, dtype=np.int32) - view.setExternalData(pysidre.TypeID.INT32_ID, 5, external1) + view.setExternalData(sidre.TypeID.INT32_ID, 5, external1) # Second external data on same view - old pin released, new pin created external2 = np.arange(10, dtype=np.int64) - view.setExternalData(pysidre.TypeID.INT64_ID, 10, external2) + view.setExternalData(sidre.TypeID.INT64_ID, 10, external2) # The second pin should be active; first pin was automatically released np.testing.assert_array_equal(view.getDataArray(), external2) @@ -564,7 +608,7 @@ def test_pin_overwrite_warning(): def test_registry_cleanup_on_explicit_destroy(): """Pins are released when Views are explicitly destroyed, preventing registry bloat.""" - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() weak_refs = [] @@ -572,7 +616,7 @@ def test_registry_cleanup_on_explicit_destroy(): external = np.arange(10, dtype=np.int32) weak_refs.append(weakref.ref(external)) view = root.createView(f"view_{i}") - view.setExternalData(pysidre.TypeID.INT32_ID, 10, external) + view.setExternalData(sidre.TypeID.INT32_ID, 10, external) del external # Pin keeps it alive del view # Don't hold view reference @@ -588,6 +632,103 @@ def test_registry_cleanup_on_explicit_destroy(): f"Only {collected_count}/{len(weak_refs)} arrays collected after explicit destroy" +def test_external_pins_released_when_datastore_destroyed(): + """Pins are released when the DataStore is destroyed without explicit destroy*(). + + This is the implicit counterpart to test_registry_cleanup_on_explicit_destroy: + the Views are torn down by the DataStore destructor (the C++ path), not by a + bound destroyView()/destroyGroup(). A weak reference on the DataStore must + still clear its pins, so the external arrays are collected and the registry + does not accumulate dangling entries. + """ + weak_refs = [] + + def build_and_drop(): + ds = sidre.DataStore() + root = ds.getRoot() + for i in range(5): + external = np.arange(10, dtype=np.int32) + weak_refs.append(weakref.ref(external)) + # Mix createView(external) and setExternalData() entry points. + if i % 2 == 0: + root.createView(f"view_{i}", external).apply(sidre.TypeID.INT32_ID, 10) + else: + root.createView(f"view_{i}").setExternalData(sidre.TypeID.INT32_ID, 10, external) + # Pins keep the arrays alive while ds is alive... + gc.collect() + assert all(ref() is not None for ref in weak_refs) + # ...and ds goes out of scope here without any explicit destroy call. + + build_and_drop() + _force_gc() + + collected = sum(1 for ref in weak_refs if ref() is None) + assert collected == len(weak_refs), \ + f"Only {collected}/{len(weak_refs)} external arrays collected after DataStore destruction" + + +def test_external_pins_released_for_nested_groups_on_datastore_destruction(): + """DataStore destruction releases pins for Views nested in child Groups too.""" + weak_refs = [] + + def build_and_drop(): + ds = sidre.DataStore() + root = ds.getRoot() + grp = root.createGroup("a/b/c") + for i in range(3): + external = np.arange(8, dtype=np.int64) + weak_refs.append(weakref.ref(external)) + grp.createView(f"deep_{i}", external).apply(sidre.TypeID.INT64_ID, 8) + gc.collect() + assert all(ref() is not None for ref in weak_refs) + + build_and_drop() + _force_gc() + + assert all(ref() is None for ref in weak_refs), \ + "Nested-Group external arrays were not released on DataStore destruction" + + +def test_external_pins_isolated_between_datastores(): + """A View* address reused across DataStores must not cross-associate pins. + + Each DataStore owns a private pin scope. Destroying one DataStore releases + only its own pins; a concurrently live DataStore is unaffected, even though + the allocator may hand out overlapping View* addresses across them. + """ + keep_alive = [] + surviving_refs = [] + + # Build and drop several DataStores in sequence, encouraging View* reuse. + for _ in range(4): + ds = sidre.DataStore() + a = np.arange(6, dtype=np.int64) + r = weakref.ref(a) + ds.getRoot().createView("v", a).apply(sidre.TypeID.INT64_ID, 6) + del a, ds + _force_gc() + # Each dropped DataStore must release its own array. + assert r() is None + + # A long-lived DataStore created afterwards (possibly at a reused address) + # must hold its own pin independently. + survivor = sidre.DataStore() + b = np.arange(6, dtype=np.int64) + surviving_refs.append(weakref.ref(b)) + survivor.getRoot().createView("v", b).apply(sidre.TypeID.INT64_ID, 6) + keep_alive.append(survivor) + del b + _force_gc() + assert surviving_refs[0]() is not None, \ + "Survivor DataStore's pin was wrongly released (cross-datastore misattribution)" + + # Cleanup releases the survivor's pin. + del survivor + keep_alive.clear() + _force_gc() + assert surviving_refs[0]() is None + + def test_multiple_concurrent_datastores(): """Multiple active DataStores with external data should not interfere with each other.""" # Create multiple DataStores simultaneously @@ -596,7 +737,7 @@ def test_multiple_concurrent_datastores(): arrays = [] for ds_idx in range(3): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() datastores.append(ds) @@ -607,7 +748,7 @@ def test_multiple_concurrent_datastores(): external = np.arange(view_idx * 10, (view_idx + 1) * 10, dtype=np.int32) ds_arrays.append(external) view = root.createView(f"view_{view_idx}") - view.setExternalData(pysidre.TypeID.INT32_ID, 10, external) + view.setExternalData(sidre.TypeID.INT32_ID, 10, external) ds_views.append(view) views.append(ds_views) @@ -650,11 +791,11 @@ def test_multiple_concurrent_datastores(): err_msg=f"After DS0 destroy: {ds_label} view{view_idx} data mismatch") # Create a new DataStore and verify it doesn't conflict - ds_new = pysidre.DataStore() + ds_new = sidre.DataStore() root_new = ds_new.getRoot() external_new = np.arange(100, 110, dtype=np.int32) view_new = root_new.createView("new_view") - view_new.setExternalData(pysidre.TypeID.INT32_ID, 10, external_new) + view_new.setExternalData(sidre.TypeID.INT32_ID, 10, external_new) # Verify new DataStore works np.testing.assert_array_equal(view_new.getDataArray(), external_new) @@ -674,8 +815,8 @@ def test_multiple_concurrent_datastores(): def test_concurrent_datastores_with_copy_move(): """Copy/move operations should work correctly with multiple concurrent DataStores.""" - ds1 = pysidre.DataStore() - ds2 = pysidre.DataStore() + ds1 = sidre.DataStore() + ds2 = sidre.DataStore() root1 = ds1.getRoot() root2 = ds2.getRoot() @@ -683,7 +824,7 @@ def test_concurrent_datastores_with_copy_move(): # Create view with external data in DS1 external1 = np.arange(10, dtype=np.int32) view1 = root1.createView("src") - view1.setExternalData(pysidre.TypeID.INT32_ID, 10, external1) + view1.setExternalData(sidre.TypeID.INT32_ID, 10, external1) # Copy view to a group in DS1 grp1 = root1.createGroup("grp1") @@ -693,7 +834,7 @@ def test_concurrent_datastores_with_copy_move(): # Create view with different external data in DS2 external2 = np.arange(20, 30, dtype=np.int64) view2 = root2.createView("other") - view2.setExternalData(pysidre.TypeID.INT64_ID, 10, external2) + view2.setExternalData(sidre.TypeID.INT64_ID, 10, external2) # Verify both DataStores maintain correct data np.testing.assert_array_equal(view1.getDataArray(), external1) @@ -711,8 +852,8 @@ def test_concurrent_datastores_with_copy_move(): def test_concurrent_datastores_registry_isolation(): """Registry should correctly isolate pins between different DataStores.""" - ds1 = pysidre.DataStore() - ds2 = pysidre.DataStore() + ds1 = sidre.DataStore() + ds2 = sidre.DataStore() external1 = np.arange(5, dtype=np.int32) external2 = np.arange(5, dtype=np.int64) @@ -722,10 +863,10 @@ def test_concurrent_datastores_registry_isolation(): # Both DataStores use external data view1 = ds1.getRoot().createView("v1") - view1.setExternalData(pysidre.TypeID.INT32_ID, 5, external1) + view1.setExternalData(sidre.TypeID.INT32_ID, 5, external1) view2 = ds2.getRoot().createView("v2") - view2.setExternalData(pysidre.TypeID.INT64_ID, 5, external2) + view2.setExternalData(sidre.TypeID.INT64_ID, 5, external2) del external1, external2 # Only pins keep them alive _force_gc() diff --git a/src/axom/sidre/tests/sidre_smoke_Py.py b/src/axom/sidre/tests/sidre_smoke_Py.py index 78bc39a943..0bb6cc8c67 100644 --- a/src/axom/sidre/tests/sidre_smoke_Py.py +++ b/src/axom/sidre/tests/sidre_smoke_Py.py @@ -4,28 +4,28 @@ # # SPDX-License-Identifier: (BSD-3-Clause) -import pysidre +import axom.sidre as sidre from conduit import Node # Python automatically calls destructor during garbage collection def test_create_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() assert True def test_valid_invalid(): - ds = pysidre.DataStore() + ds = sidre.DataStore() idx = 3 - assert idx != pysidre.InvalidIndex + assert idx != sidre.InvalidIndex name = "foo" - assert pysidre.nameIsValid(name) + assert sidre.nameIsValid(name) root = ds.getRoot() - assert root.getGroupName(idx) == pysidre.InvalidName - assert root.getGroupIndex(name) == pysidre.InvalidIndex + assert root.getGroupName(idx) == sidre.InvalidName + assert root.getGroupIndex(name) == sidre.InvalidIndex def test_conduit_in_sidre_smoke(): diff --git a/src/axom/sidre/tests/sidre_spio_Py.py b/src/axom/sidre/tests/sidre_spio_Py.py index ff42e277cb..e8649c459d 100644 --- a/src/axom/sidre/tests/sidre_spio_Py.py +++ b/src/axom/sidre/tests/sidre_spio_Py.py @@ -17,10 +17,10 @@ import pytest -import pysidre +import axom.sidre as sidre -if not pysidre.AXOM_ENABLE_MPI: - pytest.skip("pysidre built without MPI", allow_module_level=True) +if not sidre.AXOM_ENABLE_MPI: + pytest.skip("sidre built without MPI", allow_module_level=True) mpi4py = pytest.importorskip("mpi4py") from mpi4py import MPI # noqa: E402 @@ -33,9 +33,9 @@ def _shared_base(tmp_path, name): def _fill_datastore(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() - view = root.createViewAndAllocate("field", pysidre.TypeID.INT_ID, 4) + view = root.createViewAndAllocate("field", sidre.TypeID.INT_ID, 4) rank = MPI.COMM_WORLD.Get_rank() view.getDataArray()[:] = [rank, rank + 1, rank + 2, rank + 3] return ds @@ -43,21 +43,21 @@ def _fill_datastore(): def test_iomanager_legacy_use_scr_constructor(): # Preserve the previous IOManager(use_scr=False) positional API. - pysidre.IOManager(False) + sidre.IOManager(False) def test_iomanager_rejects_non_communicator(): with pytest.raises(AttributeError): - pysidre.IOManager(object()) + sidre.IOManager(object()) def test_iomanager_default_communicator(tmp_path): # No communicator argument -> MPI_COMM_WORLD (preserves the prior behavior). world = MPI.COMM_WORLD ds = _fill_datastore() - iom = pysidre.IOManager() + iom = sidre.IOManager() base = _shared_base(tmp_path, "default_comm") - iom.write(ds.getRoot(), 1, base, pysidre.Group.getDefaultIOProtocol()) + iom.write(ds.getRoot(), 1, base, sidre.Group.getDefaultIOProtocol()) world.Barrier() assert iom.getNumFilesFromRoot(base + ".root") == 1 assert iom.getNumGroupsFromRoot(base + ".root") == world.Get_size() @@ -67,12 +67,12 @@ def test_iomanager_explicit_world_communicator(tmp_path): # Passing COMM_WORLD explicitly must match the default path. world = MPI.COMM_WORLD ds = _fill_datastore() - iom = pysidre.IOManager(MPI.COMM_WORLD) + iom = sidre.IOManager(MPI.COMM_WORLD) base = _shared_base(tmp_path, "explicit_world") - iom.write(ds.getRoot(), 1, base, pysidre.Group.getDefaultIOProtocol()) + iom.write(ds.getRoot(), 1, base, sidre.Group.getDefaultIOProtocol()) world.Barrier() - ds_in = pysidre.DataStore() + ds_in = sidre.DataStore() iom.read(ds_in.getRoot(), base + ".root") arr = ds_in.getRoot().getView("field").getDataArray() rank = world.Get_rank() @@ -83,12 +83,12 @@ def test_iomanager_owned_duplicate_survives_comm_free(tmp_path): # IOManager duplicates the input communicator, so callers may free their # mpi4py communicator after construction. comm = MPI.COMM_SELF.Dup() - iom = pysidre.IOManager(comm) + iom = sidre.IOManager(comm) comm.Free() ds = _fill_datastore() base = _shared_base(tmp_path, f"freed_comm_rank{MPI.COMM_WORLD.Get_rank()}") - iom.write(ds.getRoot(), 1, base, pysidre.Group.getDefaultIOProtocol()) + iom.write(ds.getRoot(), 1, base, sidre.Group.getDefaultIOProtocol()) assert iom.getNumFilesFromRoot(base + ".root") == 1 assert iom.getNumGroupsFromRoot(base + ".root") == 1 MPI.COMM_WORLD.Barrier() @@ -106,12 +106,12 @@ def test_iomanager_split_communicator(tmp_path): try: sub_size = sub.Get_size() ds = _fill_datastore() - iom = pysidre.IOManager(sub) + iom = sidre.IOManager(sub) sub.Free() sub_freed = True # tmp_path differs per rank; rendezvous on a shared, rank-0-broadcast dir base = _shared_base(tmp_path, f"split_color{color}") - iom.write(ds.getRoot(), 1, base, pysidre.Group.getDefaultIOProtocol()) + iom.write(ds.getRoot(), 1, base, sidre.Group.getDefaultIOProtocol()) assert iom.getNumFilesFromRoot(base + ".root") == 1 assert iom.getNumGroupsFromRoot(base + ".root") == sub_size finally: @@ -122,10 +122,10 @@ def test_iomanager_split_communicator(tmp_path): def test_distributed_generate_blueprint_index(tmp_path): # The distributed generateBlueprintIndex overload is built only under # nanobind >= 2.10; skip cleanly if this build omitted it. - if not pysidre.AXOM_HAS_DISTRIBUTED_BLUEPRINT_INDEX_BINDING: + if not sidre.AXOM_HAS_DISTRIBUTED_BLUEPRINT_INDEX_BINDING: pytest.skip("generateBlueprintIndex not bound") - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() mesh = root.createGroup("mesh") coords = mesh.createGroup("coordsets/coords") diff --git a/src/axom/sidre/tests/sidre_view_Py.py b/src/axom/sidre/tests/sidre_view_Py.py index e8def9d38e..bae6074582 100644 --- a/src/axom/sidre/tests/sidre_view_Py.py +++ b/src/axom/sidre/tests/sidre_view_Py.py @@ -4,7 +4,7 @@ # # SPDX-License-Identifier: (BSD-3-Clause) -import pysidre +import axom.sidre as sidre import numpy as np NUM_BYTES_INT_32 = 4 @@ -54,11 +54,11 @@ def check_view_values(view, state, is_described, is_allocated, is_applied, lengt def test_create_views(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() - dv_0 = root.createViewAndAllocate("field0", pysidre.TypeID.INT_ID, 1) - dv_1 = root.createViewAndAllocate("field1", pysidre.TypeID.INT_ID, 1) + dv_0 = root.createViewAndAllocate("field0", sidre.TypeID.INT_ID, 1) + dv_1 = root.createViewAndAllocate("field1", sidre.TypeID.INT_ID, 1) db_0 = dv_0.getBuffer() db_1 = dv_1.getBuffer() @@ -68,7 +68,7 @@ def test_create_views(): def test_get_path_name(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() v1 = root.createView("test/a/b/v1") @@ -89,7 +89,7 @@ def test_get_path_name(): def test_create_view_from_path(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() baz = root.createView("foo/bar/baz") @@ -120,33 +120,33 @@ def check_scalar_values(view, state, is_described, is_allocated, is_applied, typ assert ndims == 1, f"{name} getShape" assert dims[0] == length, f"{name} dims[0]" - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() i1 = 1 i0view = root.createView("i0") i0view.setScalar(i1) - check_scalar_values(i0view, SCALARVIEW, True, True, True, pysidre.TypeID.INT32_ID, 1) + check_scalar_values(i0view, SCALARVIEW, True, True, True, sidre.TypeID.INT32_ID, 1) i2 = i0view.getDataInt() assert i1 == i2 i1 = 2 i1view = root.createViewScalar("i1", i1) - check_scalar_values(i1view, SCALARVIEW, True, True, True, pysidre.TypeID.INT32_ID, 1) + check_scalar_values(i1view, SCALARVIEW, True, True, True, sidre.TypeID.INT32_ID, 1) i2 = i1view.getDataInt() assert i1 == i2 s1 = "i am a string" s0view = root.createView("s0") s0view.setString(s1) - check_scalar_values(s0view, STRINGVIEW, True, True, True, pysidre.TypeID.CHAR8_STR_ID, + check_scalar_values(s0view, STRINGVIEW, True, True, True, sidre.TypeID.CHAR8_STR_ID, len(s1) + 1) s2 = s0view.getString() assert s1 == s2 s1 = "i too am a string" s1view = root.createViewString("s1", s1) - check_scalar_values(s1view, STRINGVIEW, True, True, True, pysidre.TypeID.CHAR8_STR_ID, + check_scalar_values(s1view, STRINGVIEW, True, True, True, sidre.TypeID.CHAR8_STR_ID, len(s1) + 1) s2 = s1view.getString() assert s1 == s2 @@ -162,10 +162,10 @@ def check_scalar_values(view, state, is_described, is_allocated, is_applied, typ def test_int_buffer_from_view(): elem_count = 10 - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() - dv = root.createViewAndAllocate("u0", pysidre.TypeID.INT32_ID, elem_count) + dv = root.createViewAndAllocate("u0", sidre.TypeID.INT32_ID, elem_count) data = dv.getDataArray() for i in range(elem_count): @@ -178,13 +178,13 @@ def test_int_buffer_from_view(): def test_view_dtype_support(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() dtype_pairs = [ - (pysidre.TypeID.INT8_ID, np.int8), - (pysidre.TypeID.UINT16_ID, np.uint16), - (pysidre.TypeID.FLOAT32_ID, np.float32), + (sidre.TypeID.INT8_ID, np.int8), + (sidre.TypeID.UINT16_ID, np.uint16), + (sidre.TypeID.FLOAT32_ID, np.float32), ] for idx, (type_id, expected_dtype) in enumerate(dtype_pairs): @@ -193,23 +193,23 @@ def test_view_dtype_support(): def test_detach_external_and_attach_buffer(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() external = np.array([1, 2], dtype=np.int32) - view = root.createView("external", pysidre.TypeID.INT32_ID, 2, external) + view = root.createView("external", sidre.TypeID.INT32_ID, 2, external) assert view.isExternal() assert not view.hasBuffer() view.setExternalData(None) assert view.isEmpty() - replacement = ds.createBuffer(pysidre.TypeID.INT32_ID, 4) + replacement = ds.createBuffer(sidre.TypeID.INT32_ID, 4) replacement.allocate() replacement_data = replacement.getDataArray() replacement_data[:] = [3, 4, 5, 6] - view.attachBuffer(pysidre.TypeID.INT32_ID, 4, replacement) + view.attachBuffer(sidre.TypeID.INT32_ID, 4, replacement) assert view.hasBuffer() assert not view.isExternal() @@ -217,34 +217,34 @@ def test_detach_external_and_attach_buffer(): def test_detach_buffer_and_attach_buffer(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() - original = ds.createBuffer(pysidre.TypeID.INT32_ID, 2) + original = ds.createBuffer(sidre.TypeID.INT32_ID, 2) original.allocate() original.getDataArray()[:] = [1, 2] - view = root.createView("buffered", pysidre.TypeID.INT32_ID, 2, original) + view = root.createView("buffered", sidre.TypeID.INT32_ID, 2, original) assert view.hasBuffer() view.attachBuffer(None) assert view.isEmpty() assert not view.hasBuffer() - replacement = ds.createBuffer(pysidre.TypeID.INT32_ID, 4) + replacement = ds.createBuffer(sidre.TypeID.INT32_ID, 4) replacement.allocate() replacement.getDataArray()[:] = [3, 4, 5, 6] - view.attachBuffer(pysidre.TypeID.INT32_ID, 4, replacement) + view.attachBuffer(sidre.TypeID.INT32_ID, 4, replacement) assert view.hasBuffer() assert list(view.getDataArray()) == [3, 4, 5, 6] def test_int_array_multi_view(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() - dbuff = ds.createBuffer(pysidre.TypeID.INT32_ID, 10) + dbuff = ds.createBuffer(sidre.TypeID.INT32_ID, 10) dbuff.allocate() data = dbuff.getDataArray() @@ -281,11 +281,11 @@ def test_int_array_multi_view(): def test_init_int_array_multi_view(): - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() dbuff = ds.createBuffer() - dbuff.allocate(pysidre.TypeID.INT32_ID, 10) + dbuff.allocate(sidre.TypeID.INT32_ID, 10) data = dbuff.getDataArray() for i in range(10): @@ -319,11 +319,11 @@ def test_init_int_array_multi_view(): def test_int_array_depth_view(): - ds = pysidre.DataStore() + ds = sidre.DataStore() depth_nelems = 10 total_nelems = 4 * depth_nelems - dbuff = ds.createBuffer(pysidre.TypeID.INT32_ID, total_nelems) + dbuff = ds.createBuffer(sidre.TypeID.INT32_ID, total_nelems) # Get access to our root data Group root = ds.getRoot() @@ -375,7 +375,7 @@ def test_int_array_depth_view(): def test_int_array_view_attach_buffer(): # Create our main data store - ds = pysidre.DataStore() + ds = sidre.DataStore() # Get access to our root data Group root = ds.getRoot() @@ -384,17 +384,17 @@ def test_int_array_view_attach_buffer(): # Create 2 "field" views with type and # elems elem_count = 0 - field0 = root.createView("field0", pysidre.TypeID.INT32_ID, field_nelems) + field0 = root.createView("field0", sidre.TypeID.INT32_ID, field_nelems) elem_count = elem_count + field0.getNumElements() print(f"elem_count field0 {elem_count}") - field1 = root.createView("field1", pysidre.TypeID.INT32_ID, field_nelems) + field1 = root.createView("field1", sidre.TypeID.INT32_ID, field_nelems) elem_count = elem_count + field1.getNumElements() print(f"elem_count field1 {elem_count}") assert elem_count == 2 * field_nelems # Create buffer to hold data for all fields and allocate - dbuff = ds.createBuffer(pysidre.TypeID.INT32_ID, elem_count) + dbuff = ds.createBuffer(sidre.TypeID.INT32_ID, elem_count) dbuff.allocate() assert dbuff.getNumElements() == elem_count @@ -435,13 +435,13 @@ def test_int_array_view_attach_buffer(): def test_int_array_offset_stride(): # create our main data store - ds = pysidre.DataStore() + ds = sidre.DataStore() # get access to our root data Group root = ds.getRoot() field_nelems = 20 - field0 = root.createViewAndAllocate("field0", pysidre.TypeID.DOUBLE_ID, field_nelems) + field0 = root.createViewAndAllocate("field0", sidre.TypeID.DOUBLE_ID, field_nelems) assert field0.getNumElements() == field_nelems assert field0.getBytesPerElement() == NUM_BYTES_DOUBLE assert field0.getTotalBytes() == NUM_BYTES_DOUBLE * field_nelems @@ -543,7 +543,7 @@ def test_int_array_multi_view_resize(): # into the new views # Create our main data store - ds = pysidre.DataStore() + ds = sidre.DataStore() # Get access to our root data Group root = ds.getRoot() @@ -553,7 +553,7 @@ def test_int_array_multi_view_resize(): # Create a view to hold the base buffer and allocate # we will create 4 sub views of this array - base_old = r_old.createViewAndAllocate("base_data", pysidre.TypeID.INT32_ID, 40) + base_old = r_old.createViewAndAllocate("base_data", sidre.TypeID.INT32_ID, 40) # Init the buff with values that align with the 4 subsections data = base_old.getDataArray() @@ -596,7 +596,7 @@ def test_int_array_multi_view_resize(): # Create a view to hold the base buffer base_new = r_new.createView("base_data") - base_new.allocate(pysidre.TypeID.INT32_ID, 48) + base_new.allocate(sidre.TypeID.INT32_ID, 48) base_new_data = base_new.getDataArray() for i in range(48): base_new_data[i] = 0 @@ -657,13 +657,13 @@ def test_int_array_multi_view_resize(): def test_int_array_realloc(): # Create our main data store - ds = pysidre.DataStore() + ds = sidre.DataStore() # Get access to our root data Group root = ds.getRoot() - a1 = root.createViewAndAllocate("a1", pysidre.TypeID.DOUBLE_ID, 5) - a2 = root.createViewAndAllocate("a2", pysidre.TypeID.DOUBLE_ID, 5) + a1 = root.createViewAndAllocate("a1", sidre.TypeID.DOUBLE_ID, 5) + a2 = root.createViewAndAllocate("a2", sidre.TypeID.DOUBLE_ID, 5) a1_data = a1.getDataArray() a2_data = a2.getDataArray() @@ -699,7 +699,7 @@ def test_int_array_realloc(): def test_simple_opaque(): # Create our main data store - ds = pysidre.DataStore() + ds = sidre.DataStore() # Get access to our root data Group root = ds.getRoot() @@ -713,10 +713,10 @@ def test_simple_opaque(): assert opq_view.isExternal() == True assert opq_view.isApplied() == False assert opq_view.isOpaque() == True - assert opq_view.getTypeID() == pysidre.TypeID.NO_TYPE_ID + assert opq_view.getTypeID() == sidre.TypeID.NO_TYPE_ID # Apply type to get data - opq_view.apply(pysidre.TypeID.INT32_ID, 1) + opq_view.apply(sidre.TypeID.INT32_ID, 1) opq_data = opq_view.getDataArray() assert opq_data[0] == 42 @@ -725,7 +725,7 @@ def test_simple_opaque(): def test_clear_view(): BLEN = 10 - ds = pysidre.DataStore() + ds = sidre.DataStore() root = ds.getRoot() # Create an empty view @@ -735,7 +735,7 @@ def test_clear_view(): check_view_values(view, EMPTYVIEW, False, False, False, 0) # Describe an empty view - view = root.createView("v_described", pysidre.TypeID.INT32_ID, BLEN) + view = root.createView("v_described", sidre.TypeID.INT32_ID, BLEN) check_view_values(view, EMPTYVIEW, True, False, False, BLEN) view.clear() check_view_values(view, EMPTYVIEW, False, False, False, 0) @@ -753,7 +753,7 @@ def test_clear_view(): # Allocated view, Buffer will be released nbuf = ds.getNumBuffers() - view = root.createViewAndAllocate("v_allocated", pysidre.TypeID.INT32_ID, BLEN) + view = root.createViewAndAllocate("v_allocated", sidre.TypeID.INT32_ID, BLEN) check_view_values(view, BUFFERVIEW, True, True, True, BLEN) view.clear() check_view_values(view, EMPTYVIEW, False, False, False, 0) @@ -770,12 +770,12 @@ def test_clear_view(): # Explicit buffer attached to two views dbuff = ds.createBuffer() - dbuff.allocate(pysidre.TypeID.INT32_ID, BLEN) + dbuff.allocate(sidre.TypeID.INT32_ID, BLEN) nbuf = ds.getNumBuffers() assert dbuff.getNumViews() == 0 - vother = root.createView("v_other", pysidre.TypeID.INT32_ID, BLEN) - view = root.createView("v_buffer", pysidre.TypeID.INT32_ID, BLEN) + vother = root.createView("v_other", sidre.TypeID.INT32_ID, BLEN) + view = root.createView("v_buffer", sidre.TypeID.INT32_ID, BLEN) vother.attachBuffer(dbuff) assert dbuff.getNumViews() == 1 view.attachBuffer(dbuff) @@ -790,7 +790,7 @@ def test_clear_view(): # External View ext_data = np.array(BLEN) - view = root.createView("v_external", pysidre.TypeID.INT32_ID, BLEN, ext_data) + view = root.createView("v_external", sidre.TypeID.INT32_ID, BLEN, ext_data) check_view_values(view, EXTERNALVIEW, True, True, True, BLEN) view.clear() check_view_values(view, EMPTYVIEW, False, False, False, 0) diff --git a/src/cmake/AxomMacros.cmake b/src/cmake/AxomMacros.cmake index 02a0cd6d0e..be871ebbdd 100644 --- a/src/cmake/AxomMacros.cmake +++ b/src/cmake/AxomMacros.cmake @@ -606,17 +606,37 @@ endmacro(axom_configure_file) ##------------------------------------------------------------------------------ ## axom_python_test_environment() ## -## Composes the ENVIRONMENT entry ("PYTHONPATH=::...") that provides -## the pytest paths (pytest and its dependencies) from their respective CMake cache variables. +## Composes the single ENVIRONMENT entry ("PYTHONPATH=::...") needed to +## run Axom's Python tests directly under ${Python_EXECUTABLE} without a wrapper script. ## -## Note: runtime dependencies (e.g. axom, conduit, numpy) are expected to be preprended -## via the run_python_with_axom.sh script. +## We assemble one path list here, ordered: +## +## 1. the staged Python package tree (the 'axom' package) -- runtime +## 2. conduit's python module dir -- runtime +## 3. numpy, then mpi4py (MPI configs) -- runtime +## 4. pytest and its dependencies -- test harness +## +## Axom's own package tree comes first so it is preferred over anything the +## interpreter might also provide. Entries whose cache variable is unset are skipped; +## conduit/numpy/etc. already on the interpreter's path make the corresponding entries harmless no-ops. ##------------------------------------------------------------------------------ function(axom_python_test_environment output_var) set(_paths "") - foreach(_var PY_PYTEST_DIR PY_PLUGGY_DIR PY_INICONFIG_DIR) + + # (1) staged package tree -- mirrors run_python_with_axom.sh's _PYEXT_DIR + blt_list_append(TO _paths ELEMENTS "${PROJECT_BINARY_DIR}/python") + + # (2,3) runtime dependencies + foreach(_var CONDUIT_PYTHON_MODULE_DIR PY_NUMPY_DIR PY_MPI4PY_DIR) blt_list_append(TO _paths ELEMENTS "${${_var}}" IF ${_var}) endforeach() + + # (4) test-harness dependencies + foreach(_var PY_PYTEST_DIR PY_PLUGGY_DIR PY_INICONFIG_DIR + PY_PACKAGING_DIR PY_PYGMENTS_DIR) + blt_list_append(TO _paths ELEMENTS "${${_var}}" IF ${_var}) + endforeach() + if(_paths) list(JOIN _paths ":" _joined) set(${output_var} "PYTHONPATH=${_joined}" PARENT_SCOPE) @@ -629,38 +649,57 @@ endfunction() ## axom_add_python_test(NAME [name] ## SOURCE [source] ## OUTPUT_DIR [dir] +## COMMAND [command] ## NUM_MPI_TASKS [n]) ## -## Wrapper around add_test() that handles functionality +## Wrapper around axom_add_test() that handles functionality ## that Axom applies to all python tests. +## +## When SOURCE is provided, the test file is copied to OUTPUT_DIR and run under +## pytest. When COMMAND is provided, it is registered directly as the test +## command. SOURCE and COMMAND are mutually exclusive. ##------------------------------------------------------------------------------ macro(axom_add_python_test) set(options) set(singleValueArgs NAME SOURCE OUTPUT_DIR NUM_MPI_TASKS) - set(multiValueArgs) + set(multiValueArgs COMMAND) # Parse the arguments to the macro cmake_parse_arguments(arg "${options}" "${singleValueArgs}" "${multiValueArgs}" ${ARGN}) - # Copy python test file to build - axom_configure_file ("${arg_SOURCE}" - "${arg_OUTPUT_DIR}/${arg_SOURCE}" COPYONLY) + if(arg_SOURCE AND arg_COMMAND) + message(FATAL_ERROR + "axom_add_python_test accepts either SOURCE or COMMAND, not both") + endif() - # Run unit test with pytest ("python3 -m pytest"). - # The run_python_with_axom.sh wrapper provides the runtime environment - # and the testing dependencies are injected via the test's ENVIRONMENT property when provided. - # "-p no:cacheprovider" disables caching. - set(_test_command ${PROJECT_BINARY_DIR}/bin/run_python_with_axom.sh - -m pytest -s -p no:cacheprovider ${arg_OUTPUT_DIR}/${arg_SOURCE}) - blt_add_test(NAME ${arg_NAME} - COMMAND ${_test_command} - NUM_MPI_TASKS ${arg_NUM_MPI_TASKS}) + if(arg_COMMAND) + set(_test_command ${arg_COMMAND}) + else() + if((NOT arg_SOURCE) OR (NOT arg_OUTPUT_DIR)) + message(FATAL_ERROR + "axom_add_python_test requires SOURCE and OUTPUT_DIR, or COMMAND") + endif() - set_property(TEST ${arg_NAME} - APPEND - PROPERTY ENVIRONMENT "OMPI_MCA_rmaps_base_oversubscribe=1") + # Copy python test file to build + axom_configure_file ("${arg_SOURCE}" + "${arg_OUTPUT_DIR}/${arg_SOURCE}" COPYONLY) + + # Run unit test with pytest ("python3 -m pytest"), invoked directly + # rather than through the run_python_with_axom.sh wrapper. The full + # runtime + test environment is supplied via the test's ENVIRONMENT + # property (a single combined PYTHONPATH; see axom_python_test_environment). + # Running pytest natively keeps the tests composable with IDEs/debuggers + # and removes the bash-only wrapper from the test path. + # "-p no:cacheprovider" disables caching. + set(_test_command ${Python_EXECUTABLE} + -m pytest -s -p no:cacheprovider ${arg_OUTPUT_DIR}/${arg_SOURCE}) + endif() + + axom_add_test(NAME ${arg_NAME} + COMMAND ${_test_command} + NUM_MPI_TASKS ${arg_NUM_MPI_TASKS}) axom_python_test_environment(_py_test_env) if(_py_test_env) diff --git a/src/cmake/thirdparty/SetupAxomThirdParty.cmake b/src/cmake/thirdparty/SetupAxomThirdParty.cmake index c46f1c6966..709ae46beb 100644 --- a/src/cmake/thirdparty/SetupAxomThirdParty.cmake +++ b/src/cmake/thirdparty/SetupAxomThirdParty.cmake @@ -331,8 +331,8 @@ if(EXISTS ${Python_EXECUTABLE}) ) endif() - # Check if the python environment contains the runtime dependencies for Axom's python - # conduit (Node interop) and numpy (ndarray returns). + # Check if the python environment contains the runtime dependencies + # for Axom's python conduit (Node interop) and numpy (ndarray returns). # nanobind is statically linked at build time and is located separately above. execute_process( COMMAND "${CMAKE_COMMAND}" -E env @@ -342,7 +342,7 @@ if(EXISTS ${Python_EXECUTABLE}) ERROR_QUIET ) - # Check if the python environment contains the pytest test harness, + # Check if the python environment contains the pytest test harness execute_process( COMMAND "${CMAKE_COMMAND}" -E env "${Python_EXECUTABLE}" -c "import pytest" @@ -361,6 +361,32 @@ if(EXISTS ${Python_EXECUTABLE}) ) endif() +if(AXOM_ENABLE_PYTHON_TESTS + AND (NOT PY_PYTEST_IMPORT_CODE EQUAL 0) + AND nanobind_ROOT + AND PY_PYTEST_DIR + AND PY_PLUGGY_DIR + AND PY_INICONFIG_DIR) + set(_axom_pytest_pythonpath + "${PY_PYTEST_DIR}" + "${PY_PLUGGY_DIR}" + "${PY_INICONFIG_DIR}") + foreach(_var PY_PACKAGING_DIR PY_PYGMENTS_DIR) + blt_list_append(TO _axom_pytest_pythonpath ELEMENTS "${${_var}}" IF ${_var}) + endforeach() + list(JOIN _axom_pytest_pythonpath ":" _axom_pytest_pythonpath_joined) + execute_process( + COMMAND "${CMAKE_COMMAND}" -E env + "PYTHONPATH=${_axom_pytest_pythonpath_joined}" + "${Python_EXECUTABLE}" -c "import pytest" + RESULT_VARIABLE PY_PYTEST_IMPORT_CODE + OUTPUT_QUIET + ERROR_QUIET + ) + unset(_axom_pytest_pythonpath) + unset(_axom_pytest_pythonpath_joined) +endif() + # If the python environment does not contain the required runtime modules, # check if library installation paths were provided instead. if((NOT PY_RUNTIME_IMPORT_CODE EQUAL 0) @@ -380,9 +406,9 @@ if(AXOM_ENABLE_PYTHON_TESTS AND nanobind_ROOT AND (NOT PY_PYTEST_DIR OR NOT PY_PLUGGY_DIR OR NOT PY_INICONFIG_DIR)) message(FATAL_ERROR - "Running Axom's python tests requires pytest (and its dependencies pluggy and iniconfig)." + "Running Axom's python tests requires pytest and its import-time dependencies." "\nThe library installation paths can be specified with CMake variables: " - "PY_PYTEST_DIR, PY_PLUGGY_DIR, PY_INICONFIG_DIR." + "PY_PYTEST_DIR, PY_PLUGGY_DIR, PY_INICONFIG_DIR, PY_PACKAGING_DIR, PY_PYGMENTS_DIR." "\nAlternatively, configure with AXOM_ENABLE_PYTHON_TESTS=OFF.") endif() diff --git a/src/docs/sphinx/dev_guide/component_org.rst b/src/docs/sphinx/dev_guide/component_org.rst index 75375627e7..c2c1e56116 100644 --- a/src/docs/sphinx/dev_guide/component_org.rst +++ b/src/docs/sphinx/dev_guide/component_org.rst @@ -302,22 +302,30 @@ in other languages Axom supports. Python Interfaces ==================================== -We use the nanobind library to generate Python APIs from our C++ -interface code. Nanobind is a python binding library that generates code -from a *cpp* file that describes C++ functions and their interfaces. - -Please refer to the `nanobind documentation `_ for more information. - -The python interpreter can be launched with Axom extension(s) in the PYTHONPATH -by running the convenience script:: - - ./bin/run_python_with_axom.sh - -.. note:: The Python interface requires Axom to be configured with nanobind - to build and use the interface. This requirement is different from shroud, - which generates interface files. Once shroud generates the interface - files, users are not required to configure Axom with shroud to use the - Fortran interface. +We use the `nanobind `_ library +to build Python APIs from our C++ interface code. A component's bindings are +hand-written in a nanobind translation unit (e.g., ``src/axom/sidre/nanobind_sidre.cpp``) +that describes the classes and functions to expose. +nanobind compiles this into an extension module. + +The bindings install as a Python package. Each bound component is an extension +under the ``axom`` package (for example ``axom.sidre``), with type stubs +and a ``py.typed`` marker so editors and type checkers can introspect it. +The pure-Python package scaffolding lives once under ``src/python/src/`` and is +installed by the CMake build (and, in the future, will be reused verbatim by a pip/uv wheel). + +The end-user view of the Python interface, e.g. how to install and import it, +is documented in the Sidre user guide's Python interface page. +This section covers how the bindings are built and how to add more of them. + +To build the bindings, configure Axom with nanobind (Python with the ``Development.Module`` component, +and nanobind discoverable by the interpreter). This requirement differs from Shroud, which generates Fortran interface files that do not require Shroud at build time once generated. + +.. note:: A spack environment with a view, or (in the future) the pip/uv wheel, + makes ``import axom.sidre`` work in a plain interpreter. + For running ad hoc Python scripts against a build tree, we provide a + generated ``run_python_with_axom.sh`` helper to resolve the runtime dependencies + (Conduit, NumPy, mpi4py) on ``PYTHONPATH``. .. warning:: nanobind's numpy interface does not currently support `arbitrary Python objects `_. diff --git a/src/examples/CMakeLists.txt b/src/examples/CMakeLists.txt index 9a1b6f78b4..87ecbb6e41 100644 --- a/src/examples/CMakeLists.txt +++ b/src/examples/CMakeLists.txt @@ -79,6 +79,19 @@ if (AXOM_ENABLE_EXAMPLES) ) endif() +#------------------------------------------------------------------------------ +# install 'using-with-python' example +#------------------------------------------------------------------------------ +if(AXOM_ENABLE_EXAMPLES AND NANOBIND_FOUND AND AXOM_ENABLE_SIDRE) + install( + FILES + using-with-python/README.md + using-with-python/example.py + DESTINATION + examples/axom/using-with-python + ) +endif() + #------------------------------------------------------------------------------ # configure and install 'radiuss_tutorial' example # This example requires Quest (which requires Slic, Mint, Primal and Spin) diff --git a/src/examples/using-with-python/README.md b/src/examples/using-with-python/README.md new file mode 100644 index 0000000000..7f1a5f0670 --- /dev/null +++ b/src/examples/using-with-python/README.md @@ -0,0 +1,12 @@ +# Using Axom With Python + +This example runs against an installed Axom Python package. + +From an Axom install prefix, run: + +```bash +./bin/run_python_with_axom.sh examples/axom/using-with-python/example.py +``` + +The helper script sets `PYTHONPATH` for `axom.sidre` and its Python runtime +dependencies. diff --git a/src/examples/using-with-python/example.py b/src/examples/using-with-python/example.py new file mode 100644 index 0000000000..b0f6b30e57 --- /dev/null +++ b/src/examples/using-with-python/example.py @@ -0,0 +1,26 @@ +# Copyright (c) Lawrence Livermore National Security, LLC and other +# Axom Project Contributors. See top-level LICENSE and COPYRIGHT +# files for dates and other details. +# +# SPDX-License-Identifier: (BSD-3-Clause) + +import axom.sidre as sidre +import numpy as np + + +def main(): + datastore = sidre.DataStore() + root = datastore.getRoot() + fields = root.createGroup("fields") + + values = np.arange(8, dtype=np.float64) + fields.createView("values", values).apply(sidre.TypeID.DOUBLE_ID, len(values)) + + view_data = fields.getView("values").getDataArray() + assert np.array_equal(view_data, values) + + print(f"Using installed axom.sidre {sidre.__version__}") + + +if __name__ == "__main__": + main() diff --git a/src/python/README.md b/src/python/README.md new file mode 100644 index 0000000000..92125f71c9 --- /dev/null +++ b/src/python/README.md @@ -0,0 +1,63 @@ +[comment]: # (#################################################################) +[comment]: # (Copyright Lawrence Livermore National Security, LLC and other) +[comment]: # (Axom Project Contributors. See top-level LICENSE and COPYRIGHT) +[comment]: # (files for dates and other details.) +[comment]: # +[comment]: # (# SPDX-License-Identifier: BSD-3-Clause) +[comment]: # (#################################################################) + +# Axom Python package source + +This directory (`src/python/`) holds the canonical source of Axom's Python package. +It is consumed by two independent build paths that must produce the same on-disk layout: + +1. **The CMake build (in tree).** When Axom is configured with a component's Python bindings enabled (currently Sidre), + the build stages this tree into the build directory and installs it into a `site-packages`-shaped prefix. + See `src/axom/sidre/CMakeLists.txt`: it copies the files below into `${PROJECT_BINARY_DIR}/python/` + (so the build tree is import-ready) and installs them under `AXOM_PYTHON_MODULE_INSTALL_PREFIX`. + The compiled extension (`_sidre`) and its type stub are emitted into this layout by the build; they are not checked in. + +2. **[planned] The pip/uv wheel (out of tree).** A thin, binding-only wheel built with scikit-build-core + will treat this directory as its package root (`wheel.packages = ["src/axom"]` in a sibling `pyproject.toml`), + compiling the binding translation unit against an already-installed Axom. + + +## Layout + +This is a standard "src layout" Python project root: + +``` +src/python/ + README.md <- this file + src/ + axom/ <- the 'axom' regular package + __init__.py <- top-level package metadata + py.typed <- PEP 561 marker (typed package) + sidre/ + __init__.py <- re-exports the compiled 'axom.sidre._sidre' + __init__.pyi <- package stub; re-exports '_sidre.pyi' for type checkers + (_sidre..so) <- compiled extension, produced by the build + (_sidre.pyi) <- type stub, produced by the build +``` + +Parenthesized entries are build products and are intentionally not in the repository. + +Each bound Axom component installs as a submodule of the `axom` package +(`axom.sidre`, and later `axom.quest`, `axom.primal`, ...). +A submodule is importable only when its component was enabled in the underlying Axom build. + +## What goes here vs. what does not + +- **Here:** importable pure-Python sources that are part of the installed package: + package `__init__.py` files, the `py.typed` marker, and any future pure-Python helpers or shims. +- **Not here:** the C++ binding code (each component's nanobind translation unit lives with that component, + e.g. `src/axom/sidre/nanobind_sidre.cpp`), + generated artifacts (the `.so` and `.pyi` are produced by the build), + and tests/examples (those live under the component, e.g. `src/axom/sidre/tests/*_Py.py`). + +## Notes + +- These files are installed verbatim (no template substitution). They contain no CMake-configured values. +- A `pyproject.toml` for the standalone wheel is not present yet. We will add it in the future when we add the wheel. + Until then this directory is consumed only by the CMake build. +- End-user instructions for installing and importing the bindings currently live in the Sidre user guide's "Python interface" page. diff --git a/src/python/src/axom/__init__.py b/src/python/src/axom/__init__.py new file mode 100644 index 0000000000..d4dd9dfd43 --- /dev/null +++ b/src/python/src/axom/__init__.py @@ -0,0 +1,34 @@ +# Copyright (c) Lawrence Livermore National Security, LLC and other +# Axom Project Contributors. See top-level LICENSE and COPYRIGHT +# files for dates and other details. +# +# SPDX-License-Identifier: (BSD-3-Clause) + +"""Python bindings for `LLNL Axom `_. + +Axom is a CS infrastructure library for high-performance computing applications. +Each bound Axom component is exposed as a submodule of this ``axom`` package (for example :mod:`axom.sidre`). +A submodule is importable only when the corresponding component was enabled in the underlying Axom build. +Importing a component that was not built raises :class:`ImportError` with a message naming the missing component. + +The set of submodules present in a given installation therefore mirrors +the ``AXOM_ENABLE_`` configuration of the Axom build the bindings were compiled against. +""" + +from importlib import metadata as _metadata + +# ``axom`` is a regular package (it ships this ``__init__.py``), not an +# implicit namespace package. All bound components install into this single +# package directory from one Axom build; mixing components from different +# builds is unsupported (see the build-id discussion in the bindings design +# notes). + +__all__ = ["__version__"] + + +try: + __version__ = _metadata.version("axom") +except _metadata.PackageNotFoundError: + # CMake build-tree staging does not create Python distribution metadata. + # Component modules, e.g. axom.sidre, still expose the C++ Axom version. + __version__ = "0+unknown" diff --git a/src/python/src/axom/py.typed b/src/python/src/axom/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/python/src/axom/sidre/__init__.py b/src/python/src/axom/sidre/__init__.py new file mode 100644 index 0000000000..68b2977058 --- /dev/null +++ b/src/python/src/axom/sidre/__init__.py @@ -0,0 +1,43 @@ +# Copyright (c) Lawrence Livermore National Security, LLC and other +# Axom Project Contributors. See top-level LICENSE and COPYRIGHT +# files for dates and other details. +# +# SPDX-License-Identifier: (BSD-3-Clause) + +"""Python bindings for Axom's Sidre component. + +This package re-exports the compiled ``axom.sidre._sidre`` extension module. +The extension is present only when Axom was configured with Sidre and Python bindings enabled. +If it is missing, importing :mod:`axom.sidre` raises a :class:`ImportError` that names the component, +rather than surfacing an opaque loader error. +""" + +import importlib.util as _importlib_util + +try: + from . import _sidre +except ImportError as exc: # pragma: no cover - exercised only in partial installs + if _importlib_util.find_spec(f"{__name__}._sidre") is not None: + raise + + raise ImportError( + "The 'axom.sidre' extension module ('_sidre') is not available in this " + "installation. It is built only when Axom is configured with the Sidre " + "component and Python bindings enabled " + "(AXOM_ENABLE_SIDRE=ON together with nanobind). Rebuild Axom with those " + "options, or install a build that includes them, to use axom.sidre." + ) from exc + +# Re-export the extension's public surface so ``axom.sidre.DataStore`` etc. +# resolve directly on this package. ``_sidre.__all__`` is not defined by the +# nanobind module, so fall back to a filtered ``dir()`` that drops dunders and +# the private extension handle itself. +__version__ = _sidre.__version__ + +__all__ = [_name for _name in dir(_sidre) if not _name.startswith("_")] + +globals().update({_name: getattr(_sidre, _name) for _name in __all__}) + +# ``__version__`` is conventionally public but intentionally excluded from the +# wildcard surface above (it starts with an underscore); expose it explicitly. +__all__.append("__version__") diff --git a/src/python/src/axom/sidre/__init__.pyi b/src/python/src/axom/sidre/__init__.pyi new file mode 100644 index 0000000000..14915c0f56 --- /dev/null +++ b/src/python/src/axom/sidre/__init__.pyi @@ -0,0 +1,22 @@ +# Copyright (c) Lawrence Livermore National Security, LLC and other +# Axom Project Contributors. See top-level LICENSE and COPYRIGHT +# files for dates and other details. +# +# SPDX-License-Identifier: (BSD-3-Clause) + +# Type stub for the ``axom.sidre`` package. +# +# At runtime ``__init__.py`` re-exports the compiled ``axom.sidre._sidre`` +# extension's public surface dynamically (via ``globals().update(...)``), which +# a static type checker cannot follow. This stub mirrors that re-export +# statically: ``from ._sidre import *`` pulls the typed declarations from the +# generated, adjacent ``_sidre.pyi`` so that ``axom.sidre.DataStore`` etc. +# resolve for mypy/pyright. Keep this in sync with the re-export logic in +# ``__init__.py``; the runtime module is the source of truth. + +from ._sidre import * # noqa: F401,F403 + +# ``__version__`` is conventionally public but excluded from the wildcard +# surface (it starts with an underscore), so re-export it explicitly here, +# matching ``__init__.py``. +__version__: str diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 9f3f442dc6..33cba4dff5 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -190,43 +190,48 @@ if(NANOBIND_FOUND) # Based on Conduit's run_python_with_conduit.sh.in script. #-------------------------------------------------------------------------- - # gen python helper to build directory - set(_PYEXT_DIR ${PROJECT_BINARY_DIR}/lib) + # gen python helper to build directory. + # _PYEXT_DIR is the site-packages-shaped root that holds the 'axom' package. + # Adding an entry for the Sidre bindings to PYTHONPATH entry makes 'import axom.sidre' resolve. + set(_PYEXT_DIR ${PROJECT_BINARY_DIR}/python) + set(_PYEXT_DIR_IS_RELATIVE FALSE) axom_configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_python_with_axom.sh.in" "${PROJECT_BINARY_DIR}/bin/run_python_with_axom.sh" @ONLY) - # gen python helper to install directory - set(_PYEXT_DIR ${CMAKE_INSTALL_PREFIX}/lib) + # gen python helper for install. + # Mirror the installed package root. + # Keep relative package install dirs relative in the generated script too. + # It resolves them from its own bin/ directory at runtime so `cmake --install --prefix` remains usable. + if(AXOM_PYTHON_MODULE_INSTALL_PREFIX) + set(_PYEXT_DIR ${AXOM_PYTHON_MODULE_INSTALL_PREFIX}) + if(IS_ABSOLUTE "${_PYEXT_DIR}") + set(_PYEXT_DIR_IS_RELATIVE FALSE) + else() + set(_PYEXT_DIR_IS_RELATIVE TRUE) + endif() + else() + set(_PYEXT_DIR ${CMAKE_INSTALL_PREFIX}/lib) + set(_PYEXT_DIR_IS_RELATIVE FALSE) + endif() + set(_AXOM_INSTALL_PYTHON_HELPER + "${PROJECT_BINARY_DIR}/bin/run_python_with_axom_install.sh") axom_configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_python_with_axom.sh.in" - "${CMAKE_INSTALL_PREFIX}/bin/run_python_with_axom.sh" @ONLY) + "${_AXOM_INSTALL_PYTHON_HELPER}" @ONLY) + install(PROGRAMS "${_AXOM_INSTALL_PYTHON_HELPER}" + DESTINATION bin + RENAME run_python_with_axom.sh) unset(_PYEXT_DIR) + unset(_PYEXT_DIR_IS_RELATIVE) + unset(_AXOM_INSTALL_PYTHON_HELPER) - # Smoke tests for the script. - # The wrapper provides the runtime environment only: - # Axom extensions, conduit (Node interop), numpy (ndarray returns), plus mpi4py in MPI configurations. - # nanobind is a build-time dependency (statically linked into the extensions). - # pytest/pluggy/iniconfig are test-harness dependencies. + # Smoke test for the script: verify it makes Axom's bindings and runtime dependencies importable if(AXOM_ENABLE_TESTS AND AXOM_ENABLE_SIDRE) axom_add_test( NAME run_python_with_axom_build - COMMAND ${PROJECT_BINARY_DIR}/bin/run_python_with_axom.sh -c "import pysidre, conduit, numpy") - - if(AXOM_ENABLE_PYTHON_TESTS) - # The pytest harness is provided per-test via the ENVIRONMENT property; - # verify that the wrapper + injected ENVIRONMENT combination resolves. - axom_add_test( - NAME run_python_with_axom_pytest_harness - COMMAND ${PROJECT_BINARY_DIR}/bin/run_python_with_axom.sh -c "import pytest") - axom_python_test_environment(_py_test_env) - if(_py_test_env) - set_tests_properties(run_python_with_axom_pytest_harness - PROPERTIES ENVIRONMENT "${_py_test_env}") - endif() - unset(_py_test_env) - endif() + COMMAND ${PROJECT_BINARY_DIR}/bin/run_python_with_axom.sh -c "import axom.sidre, conduit, numpy") endif() #-------------------------------------------------------------------------- @@ -239,22 +244,35 @@ if(NANOBIND_FOUND) axom_configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/convert_sidre_protocol.py" "${CMAKE_INSTALL_PREFIX}/bin/convert_sidre_protocol.py" COPYONLY) - if(AXOM_ENABLE_MPI AND AXOM_ENABLE_TESTS AND AXOM_DATA_DIR) + if(AXOM_ENABLE_PYTHON_TESTS AND AXOM_ENABLE_SIDRE) + set(_testname "convert_sidre_protocol_py") - set(box_dir "${AXOM_DATA_DIR}/quest/box_2D_r3.root") + if(AXOM_ENABLE_MPI AND AXOM_DATA_DIR) + set(box_dir "${AXOM_DATA_DIR}/quest/box_2D_r3.root") + + axom_add_python_test( + NAME ${_testname} + COMMAND ${Python_EXECUTABLE} + ${PROJECT_BINARY_DIR}/bin/convert_sidre_protocol.py + --input ${box_dir} + --output csp_output + --protocol json + --verbose + NUM_MPI_TASKS 3 + ) - set(_testname "convert_sidre_protocol_py") - axom_add_test( - NAME ${_testname} - COMMAND ${PROJECT_BINARY_DIR}/bin/run_python_with_axom.sh - ${PROJECT_BINARY_DIR}/bin/convert_sidre_protocol.py - --input ${box_dir} - --output csp_output - --protocol json - --verbose - ) + set_tests_properties(${_testname} PROPERTIES + PASS_REGULAR_EXPRESSION "Writing out datastore") + else() + axom_add_python_test( + NAME ${_testname} + COMMAND ${Python_EXECUTABLE} + ${PROJECT_BINARY_DIR}/bin/convert_sidre_protocol.py + --help + ) - set_tests_properties(${_testname} PROPERTIES - PASS_REGULAR_EXPRESSION "Writing out datastore") + set_tests_properties(${_testname} PROPERTIES + PASS_REGULAR_EXPRESSION "Sidre protocol converter") + endif() endif() endif() diff --git a/src/tools/convert_sidre_protocol.py b/src/tools/convert_sidre_protocol.py index 0a36242039..4ec3c82dfa 100644 --- a/src/tools/convert_sidre_protocol.py +++ b/src/tools/convert_sidre_protocol.py @@ -29,7 +29,7 @@ from pathlib import Path import numpy as np -import pysidre +import axom.sidre as sidre VALID_PROTOCOLS = ( "json", @@ -88,7 +88,7 @@ def parse_args() -> argparse.Namespace: # # Also initializes the data in each allocated array to zeros. # -def allocate_external_data(group: pysidre.Group, holders: list[np.ndarray], verbose: bool) -> None: +def allocate_external_data(group: sidre.Group, holders: list[np.ndarray], verbose: bool) -> None: # for each view for view in group.views(): if view.isExternal(): @@ -117,7 +117,7 @@ def allocate_external_data(group: pysidre.Group, holders: list[np.ndarray], verb # several views in the original dataset pointing to the same memory. # def modify_final_values( - view: pysidre.View, + view: sidre.View, original_size: int, retained_size: int | None = None, ) -> None: @@ -165,7 +165,7 @@ def modify_final_values( # This will be followed by at most the first max_size elements of the # original array. # -def truncate_bulk_data(group: pysidre.Group, max_size: int, verbose: bool) -> None: +def truncate_bulk_data(group: sidre.Group, max_size: int, verbose: bool) -> None: # for each view for view in group.views(): is_array = view.hasBuffer() or view.isExternal() @@ -194,8 +194,8 @@ def truncate_bulk_data(group: pysidre.Group, max_size: int, verbose: bool) -> No def main() -> int: args = parse_args() - if not pysidre.AXOM_ENABLE_MPI: - raise RuntimeError("pysidre.IOManager bindings require an MPI-enabled Axom build") + if not sidre.AXOM_ENABLE_MPI: + raise RuntimeError("sidre.IOManager bindings require an MPI-enabled Axom build") try: from mpi4py import MPI @@ -212,8 +212,8 @@ def main() -> int: comm_size = MPI.COMM_WORLD.Get_size() input_path = Path(args.input) - manager = pysidre.IOManager() - datastore = pysidre.DataStore() + manager = sidre.IOManager() + datastore = sidre.DataStore() root = datastore.getRoot() num_files = manager.getNumFilesFromRoot(str(input_path)) diff --git a/src/tools/run_python_with_axom.sh.in b/src/tools/run_python_with_axom.sh.in index 31670449f1..7819480a31 100755 --- a/src/tools/run_python_with_axom.sh.in +++ b/src/tools/run_python_with_axom.sh.in @@ -7,15 +7,27 @@ # SPDX-License-Identifier: (BSD-3-Clause) ##----------------------------------------------------------------------------- -## Convenience script that runs the python interpreter with Axom's extension(s) -## and their runtime dependencies in the PYTHONPATH: -## - Axom's extension modules (e.g. pysidre) +## Convenience script that runs the python interpreter with Axom's Python package(s) +## and their runtime dependencies already on PYTHONPATH: +## - Axom's Python package tree (the 'axom' package; e.g. axom.sidre) ## - conduit's python module (conduit::Node interop) ## - numpy (ndarray returns) ## - mpi4py (only populated in MPI-enabled configurations) ## -## Build-time (nanobind) and testing dependencies (pytest/pluggy/iniconfig) -## are intentionally NOT added here. They are expected to be added to the PYTHONPATH -## via the test's ENVIRONMENT property +## This is the supported way to run an ad hoc, non-test Python script against a +## build tree that has the bindings enabled -- it assembles conduit/numpy/mpi4py +## from their spack prefixes so a one-off script "just works" without a venv. +## +## Caveats: this script is bash-only and, since it prepends the PYTHONPATH, +## does not compose with Jupyter kernels, IDE runners, or debuggers. +## +## Build-time (nanobind) and testing dependencies (pytest/pluggy/iniconfig) +## are intentionally NOT added here. ##----------------------------------------------------------------------------- -env PYTHONPATH=@_PYEXT_DIR@:@CONDUIT_PYTHON_MODULE_DIR@:@PY_NUMPY_DIR@:@PY_MPI4PY_DIR@:$PYTHONPATH @Python_EXECUTABLE@ "$@" +_AXOM_PYEXT_DIR="@_PYEXT_DIR@" +if [ "@_PYEXT_DIR_IS_RELATIVE@" = "TRUE" ]; then + _AXOM_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + _AXOM_PYEXT_DIR="${_AXOM_SCRIPT_DIR}/../${_AXOM_PYEXT_DIR}" +fi + +env PYTHONPATH=${_AXOM_PYEXT_DIR}:@CONDUIT_PYTHON_MODULE_DIR@:@PY_NUMPY_DIR@:@PY_MPI4PY_DIR@:$PYTHONPATH @Python_EXECUTABLE@ "$@"