diff --git a/src/axom/core/Array.hpp b/src/axom/core/Array.hpp index 12319df35a..17c5e8e5c6 100644 --- a/src/axom/core/Array.hpp +++ b/src/axom/core/Array.hpp @@ -103,6 +103,32 @@ struct DefaultStoragePolicy } }; +inline HostAllocator hostAllocatorForPrimaryAllocator(int allocator_id) +{ + if(axom::isAllocatorCompatibleWithMemorySpace(allocator_id, MemorySpace::Host)) + { + return HostAllocator {allocator_id}; + } + + // Internal host scratch should not implicitly depend on the process-global + // host allocator when the primary allocator is device-only. + return HostAllocator {axom::MALLOC_ALLOCATOR_ID}; +} + +template +inline int explicitHostFallbackAllocatorID(HostAllocator host_allocator) +{ + if constexpr(SPACE == MemorySpace::Host) + { + return host_allocator.getID(); + } + else + { + AXOM_UNUSED_VAR(host_allocator); + return axom::detail::getAllocatorID(); + } +} + } // namespace detail /*! @@ -221,6 +247,11 @@ class Array : public ArrayBase>, pro IndexType capacity = 0, int allocator_id = axom::detail::getAllocatorID()); + template ::type* = nullptr> + Array(IndexType num_elements, IndexType capacity, int allocator_id, HostAllocator host_allocator); + /// \overload template >, pro IndexType capacity = 0, int allocator_id = axom::detail::getAllocatorID()); + template ::type* = nullptr> + Array(ArrayOptions::Uninitialized, + IndexType num_elements, + IndexType capacity, + int allocator_id, + HostAllocator host_allocator); + Array(const axom::StackArray& shape, int allocator_id = axom::detail::getAllocatorID()); + Array(const axom::StackArray& shape, + int allocator_id, + HostAllocator host_allocator); + /*! \brief Construct Array with row- or column-major data ordering. @@ -243,6 +287,11 @@ class Array : public ArrayBase>, pro axom::ArrayStrideOrder rowOrColumn, int allocator_id = axom::detail::getAllocatorID()); + Array(const axom::StackArray& shape, + axom::ArrayStrideOrder rowOrColumn, + int allocator_id, + HostAllocator host_allocator); + /*! \brief Construct Array with data ordering specifications. @@ -258,6 +307,12 @@ class Array : public ArrayBase>, pro const axom::StackArray& slowestDirs, int allocator_id = axom::detail::getAllocatorID()); + template + Array(const axom::StackArray& shape, + const axom::StackArray& slowestDirs, + int allocator_id, + HostAllocator host_allocator); + /*! * \brief Generic constructor for an Array of arbitrary dimension * @@ -290,6 +345,9 @@ class Array : public ArrayBase>, pro template ::type> Array(std::initializer_list elems, int allocator_id = axom::detail::getAllocatorID()); + template ::type> + Array(std::initializer_list elems, int allocator_id, HostAllocator host_allocator); + /*! * \brief Copy constructor for an Array instance */ @@ -300,7 +358,7 @@ class Array : public ArrayBase>, pro * * \note The moved-from Array is left in a valid-but-unspecified state and may be reused. */ - Array(Array&& other) noexcept; + AXOM_HOST_DEVICE Array(Array&& other) noexcept; /*! * \brief Constructor for transferring between memory spaces @@ -318,10 +376,17 @@ class Array : public ArrayBase>, pro template Array(const ArrayBase& other); + template + Array(const ArrayBase& other, HostAllocator host_allocator); + /// \overload template Array(const ArrayBase& other); + /// \overload + template + Array(const ArrayBase& other, HostAllocator host_allocator); + /*! * \brief Constructor for transferring between memory spaces, with a user- * specified allocator @@ -335,10 +400,19 @@ class Array : public ArrayBase>, pro template Array(const ArrayBase& other, int allocator_id); + template + Array(const ArrayBase& other, int allocator_id, HostAllocator host_allocator); + /// \overload template Array(const ArrayBase& other, int allocator_id); + /// \overload + template + Array(const ArrayBase& other, + int allocator_id, + HostAllocator host_allocator); + /// @} /// \name Array copy and move operators @@ -356,8 +430,9 @@ class Array : public ArrayBase>, pro this->clear(); static_cast>&>(*this) = other; m_allocator_id = other.m_allocator_id; + m_host_allocator = other.m_host_allocator; m_executeOnGPU = axom::isDeviceAllocator(m_allocator_id); - m_arrayOps = other.m_arrayOps; + m_arrayOps = OpHelper {m_allocator_id, m_executeOnGPU, m_host_allocator}; m_resize_ratio = other.m_resize_ratio; setCapacity(other.capacity()); // Use fill_range to ensure that copy constructors are invoked for each element @@ -391,8 +466,9 @@ class Array : public ArrayBase>, pro m_capacity = other.m_capacity; m_resize_ratio = other.m_resize_ratio; m_allocator_id = other.m_allocator_id; + m_host_allocator = other.m_host_allocator; m_executeOnGPU = axom::isDeviceAllocator(m_allocator_id); - m_arrayOps = other.m_arrayOps; + m_arrayOps = OpHelper {m_allocator_id, m_executeOnGPU, m_host_allocator}; other.m_data = nullptr; other.m_num_elements = 0; @@ -422,7 +498,7 @@ class Array : public ArrayBase>, pro /*! * Destructor. Frees the associated buffer. */ - ~Array(); + AXOM_HOST_DEVICE ~Array(); /// \name Array element access operators /// @{ @@ -436,6 +512,7 @@ class Array : public ArrayBase>, pro AXOM_HOST_DEVICE inline T* data() { return m_data; } AXOM_HOST_DEVICE inline const T* data() const { return m_data; } + inline int getHostAllocatorID() const { return m_host_allocator.getID(); } /// @} @@ -998,12 +1075,14 @@ class Array : public ArrayBase>, pro * \param [in] src_stride the inter-element stride between elements of the existing array * \param [in] data_space the memory space in which data has been allocated * \param [in] user_provided_allocator true if the Array's allocator ID was provided by the user + * \param [in] preserve_host_allocator true if the Array's host allocator was explicitly provided */ void initialize_from_other(const T* data, IndexType num_elements, IndexType src_stride, MemorySpace data_space, - bool user_provided_allocator); + bool user_provided_allocator, + bool preserve_host_allocator = false); /*! * \brief Updates the number of elements stored in the data array. @@ -1067,6 +1146,7 @@ class Array : public ArrayBase>, pro IndexType m_capacity = 0; double m_resize_ratio = DEFAULT_RESIZE_RATIO; int m_allocator_id = INVALID_ALLOCATOR_ID; + HostAllocator m_host_allocator {}; bool m_executeOnGPU = false; OpHelper m_arrayOps; }; @@ -1083,8 +1163,10 @@ using MCArray = Array; template Array::Array() : m_allocator_id(axom::detail::getAllocatorID()) + , m_host_allocator( + axom::detail::hostAllocatorForPrimaryAllocator(axom::detail::getAllocatorID())) , m_executeOnGPU(axom::isDeviceAllocator(m_allocator_id)) - , m_arrayOps(m_allocator_id, m_executeOnGPU) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) { } //------------------------------------------------------------------------------ @@ -1093,7 +1175,21 @@ Array::Array(const axom::StackArray>(shape) , m_allocator_id(allocator_id) - , m_arrayOps(m_allocator_id, m_executeOnGPU) + , m_host_allocator(axom::detail::hostAllocatorForPrimaryAllocator(allocator_id)) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) +{ + initialize(detail::packProduct(shape.m_data), detail::packProduct(shape.m_data), false); +} + +//------------------------------------------------------------------------------ +template +Array::Array(const axom::StackArray& shape, + int allocator_id, + HostAllocator host_allocator) + : ArrayBase>(shape) + , m_allocator_id(allocator_id) + , m_host_allocator(host_allocator) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) { initialize(detail::packProduct(shape.m_data), detail::packProduct(shape.m_data), false); } @@ -1106,7 +1202,25 @@ Array::Array(const axom::StackArray>(shape, MDMapping {shape, rowOrColumn, 1}) , m_allocator_id(allocator_id) - , m_arrayOps(m_allocator_id, m_executeOnGPU) + , m_host_allocator(axom::detail::hostAllocatorForPrimaryAllocator(allocator_id)) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) +{ + assert(rowOrColumn == axom::ArrayStrideOrder::ROW || rowOrColumn == axom::ArrayStrideOrder::COLUMN || + (DIM == 1 && rowOrColumn == axom::ArrayStrideOrder::BOTH)); + initialize(detail::packProduct(shape.m_data), detail::packProduct(shape.m_data), false); +} + +//------------------------------------------------------------------------------ +template +Array::Array(const axom::StackArray& shape, + axom::ArrayStrideOrder rowOrColumn, + int allocator_id, + HostAllocator host_allocator) + : ArrayBase>(shape, + MDMapping {shape, rowOrColumn, 1}) + , m_allocator_id(allocator_id) + , m_host_allocator(host_allocator) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) { assert(rowOrColumn == axom::ArrayStrideOrder::ROW || rowOrColumn == axom::ArrayStrideOrder::COLUMN || (DIM == 1 && rowOrColumn == axom::ArrayStrideOrder::BOTH)); @@ -1121,7 +1235,23 @@ Array::Array(const axom::StackArray>(shape, {shape, slowestDirs, 1}) , m_allocator_id(allocator_id) - , m_arrayOps(m_allocator_id, m_executeOnGPU) + , m_host_allocator(axom::detail::hostAllocatorForPrimaryAllocator(allocator_id)) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) +{ + initialize(detail::packProduct(shape.m_data), detail::packProduct(shape.m_data), false); +} + +//------------------------------------------------------------------------------ +template +template +Array::Array(const axom::StackArray& shape, + const axom::StackArray& slowestDirs, + int allocator_id, + HostAllocator host_allocator) + : ArrayBase>(shape, {shape, slowestDirs, 1}) + , m_allocator_id(allocator_id) + , m_host_allocator(host_allocator) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) { initialize(detail::packProduct(shape.m_data), detail::packProduct(shape.m_data), false); } @@ -1133,7 +1263,9 @@ Array::Array(Args... args) : ArrayBase>( StackArray {{static_cast(args)...}}) , m_allocator_id(axom::detail::getAllocatorID()) - , m_arrayOps(m_allocator_id, m_executeOnGPU) + , m_host_allocator( + axom::detail::hostAllocatorForPrimaryAllocator(axom::detail::getAllocatorID())) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) { static_assert(sizeof...(Args) == DIM, "Array size must match number of dimensions"); // Intel hits internal compiler error when casting as part of function call @@ -1149,7 +1281,9 @@ Array::Array(ArrayOptions::Uninitialized, Args... : ArrayBase>( StackArray {{static_cast(args)...}}) , m_allocator_id(axom::detail::getAllocatorID()) - , m_arrayOps(m_allocator_id, m_executeOnGPU) + , m_host_allocator( + axom::detail::hostAllocatorForPrimaryAllocator(axom::detail::getAllocatorID())) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) { static_assert(sizeof...(Args) == DIM, "Array size must match number of dimensions"); // Intel hits internal compiler error when casting as part of function call @@ -1163,19 +1297,44 @@ template template ::type*> Array::Array(IndexType num_elements, IndexType capacity, int allocator_id) : m_allocator_id(allocator_id) - , m_arrayOps(m_allocator_id, m_executeOnGPU) + , m_host_allocator(axom::detail::hostAllocatorForPrimaryAllocator(allocator_id)) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) { // If a memory space has been explicitly set for the Array object, check that // the space of the user-provided allocator matches the explicit space. - if(SPACE != MemorySpace::Dynamic && SPACE != axom::detail::getAllocatorSpace(m_allocator_id)) + if(!axom::isAllocatorCompatibleWithMemorySpace(m_allocator_id, SPACE)) { #ifdef AXOM_DEBUG std::cerr << "Incorrect allocator ID was provided for an Array object with " "explicit memory space - using default for space\n"; #endif m_allocator_id = axom::detail::getAllocatorID(); + m_host_allocator = axom::detail::hostAllocatorForPrimaryAllocator(m_allocator_id); } - m_arrayOps = OpHelper {m_allocator_id, m_executeOnGPU}; + m_arrayOps = OpHelper {m_allocator_id, m_executeOnGPU, m_host_allocator}; + initialize(num_elements, capacity); +} + +//------------------------------------------------------------------------------ +template +template ::type*> +Array::Array(IndexType num_elements, + IndexType capacity, + int allocator_id, + HostAllocator host_allocator) + : m_allocator_id(allocator_id) + , m_host_allocator(host_allocator) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) +{ + if(!axom::isAllocatorCompatibleWithMemorySpace(m_allocator_id, SPACE)) + { +#ifdef AXOM_DEBUG + std::cerr << "Incorrect allocator ID was provided for an Array object with " + "explicit memory space - using default for space\n"; +#endif + m_allocator_id = axom::detail::explicitHostFallbackAllocatorID(m_host_allocator); + } + m_arrayOps = OpHelper {m_allocator_id, m_executeOnGPU, m_host_allocator}; initialize(num_elements, capacity); } @@ -1187,19 +1346,45 @@ Array::Array(ArrayOptions::Uninitialized, IndexType capacity, int allocator_id) : m_allocator_id(allocator_id) - , m_arrayOps(m_allocator_id, m_executeOnGPU) + , m_host_allocator(axom::detail::hostAllocatorForPrimaryAllocator(allocator_id)) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) { // If a memory space has been explicitly set for the Array object, check that // the space of the user-provided allocator matches the explicit space. - if(SPACE != MemorySpace::Dynamic && SPACE != axom::detail::getAllocatorSpace(m_allocator_id)) + if(!axom::isAllocatorCompatibleWithMemorySpace(m_allocator_id, SPACE)) { #ifdef AXOM_DEBUG std::cerr << "Incorrect allocator ID was provided for an Array object with " "explicit memory space - using default for space\n"; #endif m_allocator_id = axom::detail::getAllocatorID(); + m_host_allocator = axom::detail::hostAllocatorForPrimaryAllocator(m_allocator_id); + } + m_arrayOps = OpHelper {m_allocator_id, m_executeOnGPU, m_host_allocator}; + initialize(num_elements, capacity, false); +} + +//------------------------------------------------------------------------------ +template +template ::type*> +Array::Array(ArrayOptions::Uninitialized, + IndexType num_elements, + IndexType capacity, + int allocator_id, + HostAllocator host_allocator) + : m_allocator_id(allocator_id) + , m_host_allocator(host_allocator) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) +{ + if(!axom::isAllocatorCompatibleWithMemorySpace(m_allocator_id, SPACE)) + { +#ifdef AXOM_DEBUG + std::cerr << "Incorrect allocator ID was provided for an Array object with " + "explicit memory space - using default for space\n"; +#endif + m_allocator_id = axom::detail::explicitHostFallbackAllocatorID(m_host_allocator); } - m_arrayOps = OpHelper {m_allocator_id, m_executeOnGPU}; + m_arrayOps = OpHelper {m_allocator_id, m_executeOnGPU, m_host_allocator}; initialize(num_elements, capacity, false); } @@ -1208,19 +1393,34 @@ template template Array::Array(std::initializer_list elems, int allocator_id) : m_allocator_id(allocator_id) - , m_arrayOps(m_allocator_id, m_executeOnGPU) + , m_host_allocator(axom::detail::hostAllocatorForPrimaryAllocator(allocator_id)) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) { const IndexType num_elems = static_cast(elems.size()); initialize_from_other(elems.begin(), num_elems, 1 /* stride */, MemorySpace::Dynamic, true); } +//------------------------------------------------------------------------------ +template +template +Array::Array(std::initializer_list elems, + int allocator_id, + HostAllocator host_allocator) + : m_allocator_id(allocator_id) + , m_host_allocator(host_allocator) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) +{ + initialize_from_other(elems.begin(), elems.size(), 1 /* stride */, MemorySpace::Dynamic, true, true); +} + //------------------------------------------------------------------------------ template AXOM_HOST_DEVICE Array::Array(const Array& other) : ArrayBase>( static_cast>&>(other)) , m_allocator_id(other.m_allocator_id) - , m_arrayOps(other.m_arrayOps) + , m_host_allocator(other.m_host_allocator) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) { #if defined(AXOM_DEVICE_CODE) #if defined(AXOM_DEBUG) @@ -1238,7 +1438,7 @@ AXOM_HOST_DEVICE Array::Array(const Array& other) #else this->setCapacity(other.capacity()); m_executeOnGPU = axom::isDeviceAllocator(m_allocator_id); - m_arrayOps = OpHelper {m_allocator_id, m_executeOnGPU}; + m_arrayOps = OpHelper {m_allocator_id, m_executeOnGPU, m_host_allocator}; // Use fill_range to ensure that copy constructors are invoked for each // element. MemorySpace srcSpace = SPACE; @@ -1253,24 +1453,41 @@ AXOM_HOST_DEVICE Array::Array(const Array& other) //------------------------------------------------------------------------------ template -Array::Array(Array&& other) noexcept +AXOM_HOST_DEVICE Array::Array(Array&& other) noexcept : ArrayBase>( static_cast>&&>(std::move(other))) , m_resize_ratio(0.0) - , m_arrayOps(other.m_arrayOps) + , m_allocator_id(other.m_allocator_id) + , m_host_allocator(other.m_host_allocator) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) { +#if defined(AXOM_DEVICE_CODE) + #if defined(AXOM_DEBUG) + printf( + "axom::Array: cannot move-construct on the device.\n" + "This is usually the result of capturing an array by-value in a lambda. " + "Use axom::ArrayView for value captures instead.\n"); + #endif + #if defined(__CUDA_ARCH__) + asm("trap;"); + #endif + #if defined(__HIP_DEVICE_COMPILE__) + abort(); + #endif +#else m_data = other.m_data; m_num_elements = other.m_num_elements; m_capacity = other.m_capacity; m_resize_ratio = other.m_resize_ratio; - m_allocator_id = other.m_allocator_id; + m_host_allocator = other.m_host_allocator; m_executeOnGPU = axom::isDeviceAllocator(m_allocator_id); - m_arrayOps = OpHelper {m_allocator_id, m_executeOnGPU}; + m_arrayOps = OpHelper {m_allocator_id, m_executeOnGPU, m_host_allocator}; other.m_data = nullptr; other.m_num_elements = 0; other.m_capacity = 0; other.m_resize_ratio = DEFAULT_RESIZE_RATIO; +#endif } //------------------------------------------------------------------------------ @@ -1279,7 +1496,9 @@ template Array::Array(const ArrayBase& other) : ArrayBase>(other) , m_allocator_id(static_cast(other).getAllocatorID()) - , m_arrayOps(m_allocator_id, m_executeOnGPU) + , m_host_allocator(axom::detail::hostAllocatorForPrimaryAllocator( + static_cast(other).getAllocatorID())) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) { initialize_from_other(static_cast(other).data(), static_cast(other).size(), @@ -1288,13 +1507,33 @@ Array::Array(const ArrayBase +template +Array::Array(const ArrayBase& other, + HostAllocator host_allocator) + : ArrayBase>(other) + , m_allocator_id(static_cast(other).getAllocatorID()) + , m_host_allocator(host_allocator) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) +{ + initialize_from_other(static_cast(other).data(), + static_cast(other).size(), + other.minStride(), + axom::detail::getAllocatorSpace(m_allocator_id), + false, + true); +} + //------------------------------------------------------------------------------ template template Array::Array(const ArrayBase& other) : ArrayBase>(other) , m_allocator_id(static_cast(other).getAllocatorID()) - , m_arrayOps(m_allocator_id, m_executeOnGPU) + , m_host_allocator(axom::detail::hostAllocatorForPrimaryAllocator( + static_cast(other).getAllocatorID())) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) { initialize_from_other(static_cast(other).data(), static_cast(other).size(), @@ -1303,6 +1542,24 @@ Array::Array(const ArrayBase +template +Array::Array(const ArrayBase& other, + HostAllocator host_allocator) + : ArrayBase>(other) + , m_allocator_id(static_cast(other).getAllocatorID()) + , m_host_allocator(host_allocator) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) +{ + initialize_from_other(static_cast(other).data(), + static_cast(other).size(), + other.minStride(), + axom::detail::getAllocatorSpace(m_allocator_id), + false, + true); +} + //------------------------------------------------------------------------------ template template @@ -1310,7 +1567,8 @@ Array::Array(const ArrayBase>(other) , m_allocator_id(allocatorId) - , m_arrayOps(m_allocator_id, m_executeOnGPU) + , m_host_allocator(axom::detail::hostAllocatorForPrimaryAllocator(allocatorId)) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) { int src_allocator = static_cast(other).getAllocatorID(); @@ -1321,6 +1579,27 @@ Array::Array(const ArrayBase +template +Array::Array(const ArrayBase& other, + int allocatorId, + HostAllocator host_allocator) + : ArrayBase>(other) + , m_allocator_id(allocatorId) + , m_host_allocator(host_allocator) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) +{ + int src_allocator = static_cast(other).getAllocatorID(); + + initialize_from_other(static_cast(other).data(), + static_cast(other).size(), + other.minStride(), + axom::detail::getAllocatorSpace(src_allocator), + true, + true); +} + //------------------------------------------------------------------------------ template template @@ -1328,7 +1607,8 @@ Array::Array(const ArrayBase>(other) , m_allocator_id(allocatorId) - , m_arrayOps(m_allocator_id, m_executeOnGPU) + , m_host_allocator(axom::detail::hostAllocatorForPrimaryAllocator(allocatorId)) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) { int src_allocator = static_cast(other).getAllocatorID(); @@ -1341,12 +1621,48 @@ Array::Array(const ArrayBase -Array::~Array() +template +Array::Array(const ArrayBase& other, + int allocatorId, + HostAllocator host_allocator) + : ArrayBase>(other) + , m_allocator_id(allocatorId) + , m_host_allocator(host_allocator) + , m_arrayOps(m_allocator_id, m_executeOnGPU, m_host_allocator) { + int src_allocator = static_cast(other).getAllocatorID(); + + initialize_from_other(static_cast(other).data(), + static_cast(other).size(), + other.minStride(), + axom::detail::getAllocatorSpace(src_allocator), + true, + true); +} + +//------------------------------------------------------------------------------ +template +AXOM_HOST_DEVICE Array::~Array() +{ +#if defined(AXOM_DEVICE_CODE) + #if defined(AXOM_DEBUG) + printf( + "axom::Array: cannot destroy on the device.\n" + "This is usually the result of capturing an array by-value in a lambda. " + "Use axom::ArrayView for value captures instead.\n"); + #endif + #if defined(__CUDA_ARCH__) + asm("trap;"); + #endif + #if defined(__HIP_DEVICE_COMPILE__) + abort(); + #endif +#else clear(); StoragePolicy::deallocate(m_data); m_data = nullptr; +#endif } //------------------------------------------------------------------------------ @@ -1690,6 +2006,9 @@ inline void Array::swap(Array::initialize(IndexType num_elemen capacity = (num_elements > MIN_DEFAULT_CAPACITY) ? num_elements : MIN_DEFAULT_CAPACITY; } m_executeOnGPU = axom::isDeviceAllocator(m_allocator_id); + m_arrayOps = OpHelper {m_allocator_id, m_executeOnGPU, m_host_allocator}; setCapacity(capacity); if(default_construct) { @@ -1730,11 +2050,12 @@ inline void Array::initialize_from_other( IndexType num_elements, IndexType src_stride, MemorySpace other_data_space, - bool AXOM_DEBUG_PARAM(user_provided_allocator)) + bool AXOM_DEBUG_PARAM(user_provided_allocator), + bool preserve_host_allocator) { // If a memory space has been explicitly set for the Array object, check that // the space of the user-provided allocator matches the explicit space. - if(SPACE != MemorySpace::Dynamic && SPACE != axom::detail::getAllocatorSpace(m_allocator_id)) + if(!axom::isAllocatorCompatibleWithMemorySpace(m_allocator_id, SPACE)) { #ifdef AXOM_DEBUG if(user_provided_allocator) @@ -1743,10 +2064,18 @@ inline void Array::initialize_from_other( "with explicit memory space - using default for space\n"; } #endif - m_allocator_id = axom::detail::getAllocatorID(); + if(preserve_host_allocator) + { + m_allocator_id = axom::detail::explicitHostFallbackAllocatorID(m_host_allocator); + } + else + { + m_allocator_id = axom::detail::getAllocatorID(); + m_host_allocator = axom::detail::hostAllocatorForPrimaryAllocator(m_allocator_id); + } } m_executeOnGPU = axom::isDeviceAllocator(m_allocator_id); - m_arrayOps = OpHelper {m_allocator_id, m_executeOnGPU}; + m_arrayOps = OpHelper {m_allocator_id, m_executeOnGPU, m_host_allocator}; this->setCapacity(num_elements); // Use strided copy when necessary, otherwise use efficient contiguous copy if(src_stride == 1) diff --git a/src/axom/core/ArrayBase.hpp b/src/axom/core/ArrayBase.hpp index 7b59b50897..7a777c571c 100644 --- a/src/axom/core/ArrayBase.hpp +++ b/src/axom/core/ArrayBase.hpp @@ -894,6 +894,7 @@ struct DeviceStagingBuffer T* data, IndexType begin, IndexType nelems, + HostAllocator host_allocator, bool read_from_data = false) : m_data(data) , m_begin(begin) @@ -911,8 +912,7 @@ struct DeviceStagingBuffer #if defined(AXOM_USE_CUDA) && defined(AXOM_USE_UMPIRE) if(m_deviceStage) { - int allocator_id = axom::detail::getAllocatorID(); - m_staging_buf = axom::allocate(nelems, allocator_id); + m_staging_buf = axom::allocate(nelems, host_allocator.getID()); if(read_from_data) { axom::copy(m_staging_buf, m_data + begin, sizeof(T) * nelems); @@ -921,6 +921,7 @@ struct DeviceStagingBuffer #else AXOM_UNUSED_VAR(space); AXOM_UNUSED_VAR(read_from_data); + AXOM_UNUSED_VAR(host_allocator); #endif } @@ -986,20 +987,34 @@ struct ArrayOps using StagingBuffer = DeviceStagingBuffer; public: - ArrayOps(int allocId, bool preferDevice) + AXOM_HOST_DEVICE ArrayOps() : m_host_allocator(axom::MALLOC_ALLOCATOR_ID) { } + + AXOM_HOST_DEVICE ArrayOps(int allocId, + bool preferDevice, + HostAllocator hostAllocator = HostAllocator {axom::MALLOC_ALLOCATOR_ID}) + : m_host_allocator(hostAllocator) { #if defined(AXOM_USE_GPU) && defined(AXOM_USE_UMPIRE) + #if defined(AXOM_DEVICE_CODE) + AXOM_UNUSED_VAR(allocId); + AXOM_UNUSED_VAR(preferDevice); + #else space = getAllocatorSpace(allocId); + if(space == MemorySpace::Malloc) + { + space = MemorySpace::Host; + } bool isUnifiedSpace = false; isUnifiedSpace = (space == MemorySpace::Unified || space == MemorySpace::Pinned); - #if defined(AXOM_USE_HIP) + #if defined(AXOM_USE_HIP) isUnifiedSpace = (isUnifiedSpace || space == MemorySpace::Device); - #endif + #endif if(!preferDevice && isUnifiedSpace) { space = MemorySpace::Host; } + #endif #else AXOM_UNUSED_VAR(allocId); AXOM_UNUSED_VAR(preferDevice); @@ -1042,7 +1057,7 @@ struct ArrayOps #endif // Object is neither trivially default-constructible nor trivially- // copyable. Construct instances on the host. - StagingBuffer tmp_buf(space, data, begin, nelems); + StagingBuffer tmp_buf(space, data, begin, nelems, m_host_allocator); T* data_host = tmp_buf.getStagingBuffer(); for(IndexType i = 0; i < nelems; ++i) { @@ -1074,7 +1089,7 @@ struct ArrayOps #endif // Object is not trivially-copyable, so ensure copy constructors are // called on the host. - StagingBuffer tmp_buf(space, array, begin, nelems); + StagingBuffer tmp_buf(space, array, begin, nelems, m_host_allocator); std::uninitialized_fill_n(tmp_buf.getStagingBuffer(), nelems, value); } @@ -1097,8 +1112,8 @@ struct ArrayOps { // HostOp::fill_range will handle the copy to our "staging" host buffer, // regardless of the source memory space. - StagingBuffer dst_buf(space, array, begin, nelems); - DeviceStagingBuffer src_buf(valueSpace, const_cast(values), 0, nelems, true); + StagingBuffer dst_buf(space, array, begin, nelems, m_host_allocator); + DeviceStagingBuffer src_buf(valueSpace, const_cast(values), 0, nelems, m_host_allocator, true); std::uninitialized_copy(src_buf.getStagingBuffer(), src_buf.getStagingBuffer() + nelems, dst_buf.getStagingBuffer()); @@ -1136,8 +1151,13 @@ struct ArrayOps else { // Strided case - element-by-element copy - StagingBuffer dst_buf(space, array, begin, nelems); - DeviceStagingBuffer src_buf(valueSpace, const_cast(values), 0, nelems * src_stride, true); + StagingBuffer dst_buf(space, array, begin, nelems, m_host_allocator); + DeviceStagingBuffer src_buf(valueSpace, + const_cast(values), + 0, + nelems * src_stride, + m_host_allocator, + true); T* dst = dst_buf.getStagingBuffer(); const T* src = src_buf.getStagingBuffer(); @@ -1152,8 +1172,13 @@ struct ArrayOps else { // Non-trivially copyable - use placement new with stride - StagingBuffer dst_buf(space, array, begin, nelems); - DeviceStagingBuffer src_buf(valueSpace, const_cast(values), 0, nelems * src_stride, true); + StagingBuffer dst_buf(space, array, begin, nelems, m_host_allocator); + DeviceStagingBuffer src_buf(valueSpace, + const_cast(values), + 0, + nelems * src_stride, + m_host_allocator, + true); T* dst = dst_buf.getStagingBuffer(); const T* src = src_buf.getStagingBuffer(); @@ -1206,7 +1231,7 @@ struct ArrayOps { if constexpr(!std::is_trivially_destructible_v) { - StagingBuffer tmp_buf(space, array, begin, nelems, true); + StagingBuffer tmp_buf(space, array, begin, nelems, m_host_allocator, true); T* array_host = tmp_buf.getStagingBuffer(); for(IndexType i = 0; i < nelems; i++) { @@ -1225,18 +1250,15 @@ struct ArrayOps */ void move(T* array, IndexType src_begin, IndexType src_end, IndexType dst) { -#if defined(AXOM_USE_GPU) && defined(AXOM_USE_UMPIRE) - #ifdef AXOM_USE_CUDA +#if defined(AXOM_USE_GPU) && defined(AXOM_USE_UMPIRE) && defined(AXOM_USE_CUDA) // CUDA-only: we require non-trivial types to be trivially-relocatable. // This enables us to do simple memcpys for move operations. bool presume_trivially_relocatable = (space == MemorySpace::Device); - #else - constexpr bool presume_trivially_relocatable = false; - #endif - if(std::is_trivially_copyable_v || presume_trivially_relocatable) + if(space == MemorySpace::Device && + (std::is_trivially_copyable_v || presume_trivially_relocatable)) { - // Since this memory is on the device-side, we copy it to a temporary buffer - // first. + // Device-only CUDA memory cannot be shifted from the host in-place. + // Copy it through a temporary device buffer instead. IndexType nelems = src_end - src_begin; T* tmp_buf = axom::allocate(nelems, axom::execution_space::allocatorID()); axom::copy(tmp_buf, array + src_begin, nelems * sizeof(T)); @@ -1299,6 +1321,9 @@ struct ArrayOps destroy(values, 0, nelems); } } + +private: + HostAllocator m_host_allocator; }; template diff --git a/src/axom/core/ArrayView.hpp b/src/axom/core/ArrayView.hpp index 134909b593..f047527970 100644 --- a/src/axom/core/ArrayView.hpp +++ b/src/axom/core/ArrayView.hpp @@ -379,7 +379,7 @@ AXOM_HOST_DEVICE ArrayView::ArrayView(ArrayBase::ArrayView( #if !defined(AXOM_DEVICE_CODE) && defined(AXOM_DEBUG) // If it's not dynamic, the allocator ID from the argument array has to match the template param. // If that's not the case then things have gone horribly wrong somewhere. - if(SPACE != MemorySpace::Dynamic && SPACE != axom::detail::getAllocatorSpace(m_allocator_id)) + if(!axom::isAllocatorCompatibleWithMemorySpace(m_allocator_id, SPACE)) { std::cerr << "Input argument allocator does not match the explicitly " "provided memory space\n"; diff --git a/src/axom/core/CMakeLists.txt b/src/axom/core/CMakeLists.txt index 958a347058..f45ac75509 100644 --- a/src/axom/core/CMakeLists.txt +++ b/src/axom/core/CMakeLists.txt @@ -30,6 +30,7 @@ set(core_headers utilities/CommandLineUtilities.hpp utilities/ConstexprAssert.hpp utilities/FileUtilities.hpp + utilities/MemoryTesting.hpp utilities/RAII.hpp utilities/Sorting.hpp utilities/StringUtilities.hpp diff --git a/src/axom/core/FlatMap.hpp b/src/axom/core/FlatMap.hpp index fc79347c5b..aaea623828 100644 --- a/src/axom/core/FlatMap.hpp +++ b/src/axom/core/FlatMap.hpp @@ -160,9 +160,11 @@ class FlatMap : detail::flat_map::SequentialLookupPolicy(m_metadata.view(), - other.m_buckets.view(), - m_buckets.view()); + detail::flat_map::copyBuckets( + m_metadata.view(), + other.m_buckets.view(), + m_buckets.view(), + HostAllocator {m_buckets.getHostAllocatorID()}); } /*! @@ -207,9 +209,11 @@ class FlatMap : detail::flat_map::SequentialLookupPolicy(m_metadata.view(), - other.m_buckets.view(), - m_buckets.view()); + detail::flat_map::copyBuckets( + m_metadata.view(), + other.m_buckets.view(), + m_buckets.view(), + HostAllocator {m_buckets.getHostAllocatorID()}); } /// \brief Destructor for a FlatMap instance. @@ -218,8 +222,10 @@ class FlatMap : detail::flat_map::SequentialLookupPolicy 0) { - detail::flat_map::destroyBuckets(m_metadata.view(), - m_buckets.view()); + detail::flat_map::destroyBuckets( + m_metadata.view(), + m_buckets.view(), + HostAllocator {m_buckets.getHostAllocatorID()}); } // Unlike in clear() we don't need to reset metadata here. @@ -384,7 +390,10 @@ class FlatMap : detail::flat_map::SequentialLookupPolicy(m_metadata.view(), m_buckets.view()); + detail::flat_map::destroyBuckets( + m_metadata.view(), + m_buckets.view(), + HostAllocator {m_buckets.getHostAllocatorID()}); // Also reset metadata. IndexType numGroupsRounded = IndexType {1} << m_numGroups2; diff --git a/src/axom/core/detail/FlatMapOps.hpp b/src/axom/core/detail/FlatMapOps.hpp index 3ae2dea360..0e86c03379 100644 --- a/src/axom/core/detail/FlatMapOps.hpp +++ b/src/axom/core/detail/FlatMapOps.hpp @@ -34,8 +34,16 @@ inline void setSentinel(axom::ArrayView metadata) } template > -inline void destroyBuckets(axom::ArrayView metadata, axom::ArrayView buckets) +inline void destroyBuckets(axom::ArrayView metadata, + axom::ArrayView buckets, + HostAllocator host_allocator) { +#if !defined(AXOM_USE_UMPIRE) || !defined(AXOM_USE_CUDA) + // Note: HIP can access device memory from the host and does not need special + // handling - we just defer to the host path in all cases. + AXOM_UNUSED_VAR(host_allocator); +#endif + if(std::is_trivially_destructible::value) { // Nothing to do. @@ -43,8 +51,6 @@ inline void destroyBuckets(axom::ArrayView metadata, axom::ArrayVie } #if defined(AXOM_USE_UMPIRE) && defined(AXOM_USE_CUDA) - // Note: HIP can access device memory from the host and does not need special - // handling - we just defer to the host path in all cases. MemorySpace space = getAllocatorSpace(metadata.getAllocatorID()); // CUDA-only: buckets located in device-only memory and non-trivially // destructible. We'll need to "relocate" the objects to the host to @@ -56,9 +62,8 @@ inline void destroyBuckets(axom::ArrayView metadata, axom::ArrayVie axom::Array metadata_host; if(space == MemorySpace::Device) { - int host_allocator_id = axom::execution_space::allocatorID(); - metadata_host = axom::Array(metadata, host_allocator_id); - buckets_host = axom::Array(buckets, host_allocator_id); + metadata_host = axom::Array(metadata, host_allocator.getID(), host_allocator); + buckets_host = axom::Array(buckets, host_allocator.getID(), host_allocator); metadata = metadata_host.view(); buckets = buckets_host.view(); } @@ -74,8 +79,15 @@ inline void destroyBuckets(axom::ArrayView metadata, axom::ArrayVie template > inline void copyBuckets(axom::ArrayView metadata, axom::ArrayView from_buckets, - axom::ArrayView to_buckets) + axom::ArrayView to_buckets, + HostAllocator host_allocator) { +#if !defined(AXOM_USE_UMPIRE) || !defined(AXOM_USE_CUDA) + // Note: HIP can access device memory from the host and does not need special + // handling - we just defer to the host path in all cases. + AXOM_UNUSED_VAR(host_allocator); +#endif + if(std::is_trivially_copyable::value) { axom::copy(to_buckets.data(), from_buckets.data(), sizeof(StoragePair) * from_buckets.size()); @@ -84,25 +96,21 @@ inline void copyBuckets(axom::ArrayView metadata, axom::ArrayView to_buckets_stage = to_buckets; #if defined(AXOM_USE_UMPIRE) && defined(AXOM_USE_CUDA) - // Note: HIP can access device memory from the host and does not need special - // handling - we just defer to the host path in all cases. - // Non-trivially copyable: // "Relocate" to the host to call host-based copy constructor. MemorySpace meta_space = getAllocatorSpace(metadata.getAllocatorID()); axom::Array metadata_host; if(meta_space == MemorySpace::Device) { - int host_allocator_id = axom::execution_space::allocatorID(); - metadata_host = axom::Array(metadata, host_allocator_id); + metadata_host = axom::Array(metadata, host_allocator.getID(), host_allocator); metadata = metadata_host.view(); } MemorySpace from_space = getAllocatorSpace(from_buckets.getAllocatorID()); axom::Array from_buckets_host; if(from_space == MemorySpace::Device) { - int host_allocator_id = axom::execution_space::allocatorID(); - from_buckets_host = axom::Array(from_buckets, host_allocator_id); + from_buckets_host = + axom::Array(from_buckets, host_allocator.getID(), host_allocator); from_buckets = from_buckets_host.view(); } @@ -110,8 +118,7 @@ inline void copyBuckets(axom::ArrayView metadata, axom::Array to_buckets_host; if(to_space == MemorySpace::Device) { - int host_allocator_id = axom::execution_space::allocatorID(); - to_buckets_host = axom::Array(to_buckets, host_allocator_id); + to_buckets_host = axom::Array(to_buckets, host_allocator.getID(), host_allocator); to_buckets_stage = to_buckets_host.view(); } #endif diff --git a/src/axom/core/docs/sphinx/core_memory_management.rst b/src/axom/core/docs/sphinx/core_memory_management.rst new file mode 100644 index 0000000000..ac5129c444 --- /dev/null +++ b/src/axom/core/docs/sphinx/core_memory_management.rst @@ -0,0 +1,181 @@ +.. ## Copyright (c) Lawrence Livermore National Security, LLC and other +.. ## Axom Project Contributors. See top-level LICENSE and COPYRIGHT +.. ## files for dates and other details. +.. ## +.. ## SPDX-License-Identifier: (BSD-3-Clause) + +****************************************************** +Core memory management +****************************************************** + +The Axom Core component provides mechanisms to control which memory spaces +are used for allocations to support code execution on CPUs and GPUs and integration +with tools like `Umpire`_. Memory spaces are specified in the Core interface +using enum values: + +.. literalinclude:: ../../memory_management.hpp + :start-after: _memory_space_start + :end-before: _memory_space_end + :language: C++ + +For CPU-accessible memory, Axom uses the concepts of a **global allocator** and +a **host allocator**. Referring to the enum class values above, the Axom +global default allocator is the default for ``MemorySpace::Dynamic`` and for +interface routines that do not specify an allocator; for example, many +``axom::Array`` and ``axom::ArrayView`` APIs. + +The Axom host allocator is a process-wide *default* used by legacy convenience +paths that resolve ``MemorySpace::Host`` through global state. New and updated +APIs prefer that host allocation intent be expressed explicitly via the +``axom::HostAllocator`` wrapper type, rather than by relying on the process +default. + +Axom must be configured with Umpire enabled to have access to GPU memory +resources. Whether or not Axom is configured with Umpire also controls +default behavior for CPU memory allocations. Specifically, when Umpire is +enabled: + + * the default host allocator is ``malloc`` (i.e., ``MemorySpace::Malloc``) + * the default global allocator is Umpire's default allocator + +When Umpire is disabled: + + * the default host allocator and the default global allocator both refer to + ``MemorySpace::Malloc`` + +.. note:: The Axom default host allocator is ``axom::MemorySpace::Malloc`` regardless of whether Axom is configured with Umpire. + +The separation of host and global allocation in Axom is intentional because, +in Umpire-enabled builds, the global default may be device, unified, or some +other Umpire allocator, and the host allocator can still be Axom's malloc or +Umpire's Host allocator. + +Explicit host allocation +^^^^^^^^^^^^^^^^^^^^^^^^ + +When an API argument semantically means "host-resident storage" (including host +staging/scratch for device operations), prefer passing an ``axom::HostAllocator`` +object. This makes host allocation behavior independent of process-global defaults and +reduces sensitivity to initialization order. + +The following illustrates the general pattern:: + + axom::HostAllocator hostAlloc {axom::MALLOC_ALLOCATOR_ID}; + // Pass hostAlloc into APIs that allocate host memory or host staging. + +For APIs that need both an execution-space allocator and host staging, keep the +two choices separate:: + + int dataAllocId = axom::execution_space>::allocatorID(); + axom::HostAllocator hostAlloc {axom::MALLOC_ALLOCATOR_ID}; + // Pass dataAllocId for primary storage and hostAlloc for host scratch/staging. + +Several Core APIs now expose explicit host allocator overloads. For example, +``axom::Array`` constructors can receive both the primary allocator ID and an +``axom::HostAllocator``, and ``axom::fill()`` can receive a host allocator for +temporary host scratch used when filling non-host allocations:: + + axom::fill(devicePtr, numValues, value, hostAlloc); + +Overloads that do not receive ``axom::HostAllocator`` remain available for +compatibility. They forward through Axom's current default host allocator and +should be treated as legacy convenience paths in new production code. + +Changing the default host and global allocators (legacy) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The contents of this section refer to Axom *legacy* convenience routines, +meaning those that do not take an explicit host allocator argument. +The default host allocator controls what Axom uses when code asks for +``MemorySpace::Host`` through these legacy convenience paths. This is separate from +the global default allocator used for ``MemorySpace::Dynamic``. + +In practice, that means: + +* ``axom::setDefaultHostAllocator(...)`` changes where legacy + ``MemorySpace::Host`` allocations go. +* ``axom::setDefaultAllocator(...)`` changes the global default allocator for + ``MemorySpace::Dynamic``. +* Changing one does not automatically change the other. + +Axom host allocator choice can still be selected at run time for compatibility +paths. For example, to switch between Axom's malloc-backed host allocator and +the platform host allocator:: + + // set Axom host allocator to Axom malloc + axom::setDefaultHostAllocator(axom::MemorySpace::Malloc); + + // set Axom host allocator to Umpire Host allocator + axom::setDefaultHostAllocator(axom::MemorySpace::Host); + +You can also inspect the current legacy selection:: + + int hostAllocId = axom::getDefaultHostAllocatorID(); + int hostSpaceId = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Host); + + // hostAllocId and hostSpaceId refer to the same allocator. + +If you need to preserve legacy ``MemorySpace::Host`` behavior in an +Umpire-enabled build, you can resolve the host resource allocator ID yourself +and install it as Axom's default host allocator:: + + // set Axom host allocator to an explicitly chosen Umpire allocator + int hostId = + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); + axom::setDefaultHostAllocator(hostId); + +The following example shows that the host allocator and global allocator are +independent. Here, ``MemorySpace::Dynamic`` is set to Umpire Host while +``MemorySpace::Host`` still uses Axom malloc:: + + axom::setDefaultHostAllocator(axom::MemorySpace::Malloc); + axom::setDefaultAllocator(axom::MemorySpace::Host); + + int dynamicId = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Dynamic); + int hostId = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Host); + + // dynamicId is the Umpire Host allocator + // hostId is axom::MALLOC_ALLOCATOR_ID + +Once Axom has made an allocation from the current host allocator, that host +allocator selection is fixed for the remainder of the process. Configure the +desired host allocator before creating ``MemorySpace::Host`` allocations or +other allocations that use the current Axom host allocator. For example:: + + axom::setDefaultHostAllocator(axom::MemorySpace::Host); + + int* values = + axom::allocate(16, axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Host)); + + // After this allocation, changing the default host allocator will abort. + // For example, axom::setDefaultHostAllocator(axom::MemorySpace::Malloc); + // will abort when Axom is configured with Umpire since, in that case, + // axom::MemorySpace::Host is the Umpire Host allocator, which is different than + // Axom's malloc allocator. + +Similarly, the Axom global allocator can be changed. For example:: + + // set Axom global allocator to Umpire Host + axom::setDefaultAllocator(axom::MemorySpace::Host); + + // set Axom global allocator to Umpire Unified memory allocator + axom::setDefaultAllocator(axom::MemorySpace::Unified); + + // set Axom global allocator to an explicitly chosen Umpire allocator + int allocId = + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Pinned); + axom::setDefaultAllocator(allocId); + + // Since Axom exposes additional MemorySpace enum values when configured with Umpire, + // the previous example can also be done this way + int allocId = + axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Pinned); + axom::setDefaultAllocator(allocId); + +One important thing to note is that, when Umpire is enabled, you cannot set +the global allocator to Axom malloc:: + + // currently, can't do this -- code will abort + axom::setDefaultAllocator(axom::MALLOC_ALLOCATOR_ID); + +.. _Umpire: https://umpire.readthedocs.io diff --git a/src/axom/core/docs/sphinx/index.rst b/src/axom/core/docs/sphinx/index.rst index 44ccd4c47d..3b203e9fc9 100644 --- a/src/axom/core/docs/sphinx/index.rst +++ b/src/axom/core/docs/sphinx/index.rst @@ -31,6 +31,7 @@ Doxygen generated API documentation can be found here: `API documentation <../.. :caption: Contents :maxdepth: 1 + core_memory_management core_numerics core_utilities core_containers diff --git a/src/axom/core/examples/core_array_perf.cpp b/src/axom/core/examples/core_array_perf.cpp index 4ed0ba1dbe..f0eeb0df47 100644 --- a/src/axom/core/examples/core_array_perf.cpp +++ b/src/axom/core/examples/core_array_perf.cpp @@ -159,30 +159,44 @@ InputParams params; //!@brief Return allocator id suitable for the given runtime policy. int allocatorIdFromPolicy(axom::runtime_policy::Policy policy) { + int allocatorID = axom::getDefaultAllocatorID(); + +#if !defined(AXOM_USE_UMPIRE) AXOM_UNUSED_VAR(policy); -#if defined(AXOM_USE_UMPIRE) - int allocatorID = policy == axom::runtime_policy::Policy::seq - ? axom::detail::getAllocatorID() - : +#else + + const axom::HostAllocator hostAllocator {axom::execution_space::allocatorID()}; + + allocatorID = axom::INVALID_ALLOCATOR_ID; + + if(policy == axom::runtime_policy::Policy::seq) + { + allocatorID = hostAllocator.getID(); + } + #if defined(AXOM_RUNTIME_POLICY_USE_OPENMP) - policy == axom::runtime_policy::Policy::omp - ? axom::detail::getAllocatorID() - : + if(policy == axom::runtime_policy::Policy::omp) + { + allocatorID = hostAllocator.getID(); + } #endif + #if defined(AXOM_RUNTIME_POLICY_USE_CUDA) - policy == axom::runtime_policy::Policy::cuda - ? axom::detail::getAllocatorID() - : + if(policy == axom::runtime_policy::Policy::cuda) + { + allocatorID = axom::detail::getAllocatorID(); + } #endif + #if defined(AXOM_RUNTIME_POLICY_USE_HIP) - policy == axom::runtime_policy::Policy::hip - ? axom::detail::getAllocatorID() - : + if(policy == axom::runtime_policy::Policy::hip) + { + allocatorID = axom::detail::getAllocatorID(); + } #endif - axom::INVALID_ALLOCATOR_ID; -#else - int allocatorID = axom::getDefaultAllocatorID(); -#endif + +#endif // AXOM_USE_UMPIRE + return allocatorID; } @@ -204,11 +218,18 @@ class MDMappingPerfTester { m_allocatorId = allocatorIdFromPolicy(params.runtimePolicy); #ifdef AXOM_USE_UMPIRE - umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); - umpire::Allocator allocator = rm.getAllocator(m_allocatorId); - std::cout << axom::fmt::format("Allocator id: {}, Umpire memory space {}", - m_allocatorId, - allocator.getName()) + std::string name; + if(m_allocatorId == axom::MALLOC_ALLOCATOR_ID) + { + name = "Malloc"; + } + else + { + umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); + name = rm.getAllocator(m_allocatorId).getName(); + } + + std::cout << axom::fmt::format("Allocator id: {}, Umpire memory space {}", m_allocatorId, name) << std::endl; #else std::cout << axom::fmt::format("Allocator id: {}, default memory space", m_allocatorId) diff --git a/src/axom/core/examples/core_containers.cpp b/src/axom/core/examples/core_containers.cpp index 0303a4dfeb..4f45e3a8cb 100644 --- a/src/axom/core/examples/core_containers.cpp +++ b/src/axom/core/examples/core_containers.cpp @@ -315,8 +315,9 @@ void demoArrayDevice() axom::Array C_explicit_unified_copy(C_device, unified_alloc_id); // Note that if an allocator ID is incompatible with a memory space, the default - // allocator ID for that memory space is used. Both of these examples will copy - // memory to the host: + // allocator ID for that memory space is used. These compile-time + // MemorySpace::Host constructors are legacy convenience behavior, so both of + // these examples will copy memory to the host: axom::Array C_use_host_alloc(C_device); // The below will also print a warning in debug mode: axom::Array C_use_host_alloc_2(C_device, unified_alloc_id); @@ -334,6 +335,9 @@ void demoArrayDevice() add<<<1, 1>>>(A_dynamic, B_unified, C_device); // Since our result array is in device memory, we copy it to host memory so we can view it. + // This compile-time MemorySpace::Host conversion is a compatibility + // convenience; new code that wants explicit host allocator control should + // prefer the dynamic-space copy constructor shown below. axom::Array C_host = C_device; std::cout << "Array C_host = " << C_host << std::endl; diff --git a/src/axom/core/execution/internal/cuda_exec.hpp b/src/axom/core/execution/internal/cuda_exec.hpp index 16b30fdf5c..b0332ee430 100644 --- a/src/axom/core/execution/internal/cuda_exec.hpp +++ b/src/axom/core/execution/internal/cuda_exec.hpp @@ -62,10 +62,7 @@ struct execution_space> AXOM_HOST_DEVICE static constexpr bool onDevice() noexcept { return true; } AXOM_HOST_DEVICE static constexpr char* name() noexcept { return (char*)"[CUDA_EXEC]"; } - static int allocatorID() noexcept - { - return axom::getUmpireResourceAllocatorID(umpire::resource::Device); - } + static int allocatorID() noexcept { return axom::getAllocatorIDFromMemorySpace(memory_space); } AXOM_HOST_DEVICE static constexpr runtime_policy::Policy runtimePolicy() noexcept { return runtime_policy::Policy::cuda; @@ -103,10 +100,7 @@ struct execution_space> AXOM_HOST_DEVICE static constexpr bool valid() noexcept { return true; } AXOM_HOST_DEVICE static constexpr bool onDevice() noexcept { return true; } AXOM_HOST_DEVICE static constexpr char* name() noexcept { return (char*)"[CUDA_EXEC] (async)"; } - static int allocatorID() noexcept - { - return axom::getUmpireResourceAllocatorID(umpire::resource::Device); - } + static int allocatorID() noexcept { return axom::getAllocatorIDFromMemorySpace(memory_space); } AXOM_HOST_DEVICE static constexpr runtime_policy::Policy runtimePolicy() noexcept { return runtime_policy::Policy::cuda; diff --git a/src/axom/core/execution/internal/hip_exec.hpp b/src/axom/core/execution/internal/hip_exec.hpp index 87418778d7..2ca4e1d249 100644 --- a/src/axom/core/execution/internal/hip_exec.hpp +++ b/src/axom/core/execution/internal/hip_exec.hpp @@ -60,10 +60,7 @@ struct execution_space> AXOM_HOST_DEVICE static constexpr bool onDevice() noexcept { return true; } AXOM_HOST_DEVICE static constexpr char* name() noexcept { return (char*)"[HIP_EXEC]"; } - static int allocatorID() noexcept - { - return axom::getUmpireResourceAllocatorID(umpire::resource::Device); - } + static int allocatorID() noexcept { return axom::getAllocatorIDFromMemorySpace(memory_space); } AXOM_HOST_DEVICE static constexpr runtime_policy::Policy runtimePolicy() noexcept { return runtime_policy::Policy::hip; @@ -101,10 +98,7 @@ struct execution_space> AXOM_HOST_DEVICE static constexpr bool valid() noexcept { return true; } AXOM_HOST_DEVICE static constexpr bool onDevice() noexcept { return true; } AXOM_HOST_DEVICE static constexpr char* name() noexcept { return (char*)"[HIP_EXEC] (async)"; } - static int allocatorID() noexcept - { - return axom::getUmpireResourceAllocatorID(umpire::resource::Device); - } + static int allocatorID() noexcept { return axom::getAllocatorIDFromMemorySpace(memory_space); } AXOM_HOST_DEVICE static constexpr runtime_policy::Policy runtimePolicy() noexcept { return runtime_policy::Policy::hip; diff --git a/src/axom/core/execution/internal/omp_exec.hpp b/src/axom/core/execution/internal/omp_exec.hpp index 3da39efb96..f822fbfa0b 100644 --- a/src/axom/core/execution/internal/omp_exec.hpp +++ b/src/axom/core/execution/internal/omp_exec.hpp @@ -16,11 +16,6 @@ #error OMP_EXEC requires an OpenMP enabled RAJA #endif -// Umpire includes -#ifdef AXOM_USE_UMPIRE - #include "umpire/Umpire.hpp" -#endif - namespace axom { /*! @@ -41,36 +36,26 @@ struct execution_space using atomic_policy = RAJA::omp_atomic; using sync_policy = RAJA::omp_synchronize; -#ifdef AXOM_USE_UMPIRE static constexpr MemorySpace memory_space = MemorySpace::Host; -#else - static constexpr MemorySpace memory_space = MemorySpace::Dynamic; -#endif AXOM_HOST_DEVICE static constexpr bool async() noexcept { return false; } AXOM_HOST_DEVICE static constexpr bool valid() noexcept { return true; } AXOM_HOST_DEVICE static constexpr bool onDevice() noexcept { return false; } AXOM_HOST_DEVICE static constexpr char* name() noexcept { return (char*)"[OMP_EXEC]"; } - static int allocatorID() noexcept - { -#ifdef AXOM_USE_UMPIRE - return axom::getUmpireResourceAllocatorID(umpire::resource::Host); -#else - return axom::getDefaultAllocatorID(); -#endif - } + static int allocatorID() noexcept { return axom::getAllocatorIDFromMemorySpace(memory_space); } AXOM_HOST_DEVICE static constexpr runtime_policy::Policy runtimePolicy() noexcept { return runtime_policy::Policy::omp; } static bool usesMemorySpace(axom::MemorySpace m) noexcept { - return m == MemorySpace::Dynamic || m == MemorySpace::Malloc -#ifdef AXOM_USE_UMPIRE - || m == MemorySpace::Host || m == MemorySpace::Unified +#if defined(AXOM_USE_UMPIRE) + return m == MemorySpace::Dynamic || m == MemorySpace::Malloc || m == MemorySpace::Host || + (m == MemorySpace::Unified && axom::isMemorySpaceAvailable(MemorySpace::Unified)); +#else + return m == MemorySpace::Dynamic || m == MemorySpace::Malloc || m == MemorySpace::Host; #endif - ; } static bool usesAllocId(int allocId) noexcept { diff --git a/src/axom/core/execution/internal/seq_exec.hpp b/src/axom/core/execution/internal/seq_exec.hpp index ce9bac758b..8db6ba339d 100644 --- a/src/axom/core/execution/internal/seq_exec.hpp +++ b/src/axom/core/execution/internal/seq_exec.hpp @@ -14,11 +14,6 @@ #include "RAJA/RAJA.hpp" #endif -// Umpire includes -#ifdef AXOM_USE_UMPIRE - #include "umpire/Umpire.hpp" -#endif - namespace axom { /*! @@ -51,36 +46,26 @@ struct execution_space using sync_policy = void; -#ifdef AXOM_USE_UMPIRE static constexpr MemorySpace memory_space = MemorySpace::Host; -#else - static constexpr MemorySpace memory_space = MemorySpace::Dynamic; -#endif AXOM_HOST_DEVICE static constexpr bool async() noexcept { return false; } AXOM_HOST_DEVICE static constexpr bool valid() noexcept { return true; } AXOM_HOST_DEVICE static constexpr bool onDevice() noexcept { return false; } AXOM_HOST_DEVICE static constexpr char* name() noexcept { return (char*)"[SEQ_EXEC]"; } - static int allocatorID() noexcept - { -#ifdef AXOM_USE_UMPIRE - return axom::getUmpireResourceAllocatorID(umpire::resource::Host); -#else - return axom::getDefaultAllocatorID(); -#endif - } + static int allocatorID() noexcept { return axom::getAllocatorIDFromMemorySpace(memory_space); } AXOM_HOST_DEVICE static constexpr runtime_policy::Policy runtimePolicy() noexcept { return runtime_policy::Policy::seq; } static bool usesMemorySpace(axom::MemorySpace m) noexcept { - return m == MemorySpace::Dynamic || m == MemorySpace::Malloc -#ifdef AXOM_USE_UMPIRE - || m == MemorySpace::Host || m == MemorySpace::Unified +#if defined(AXOM_USE_UMPIRE) + return m == MemorySpace::Dynamic || m == MemorySpace::Malloc || m == MemorySpace::Host || + (m == MemorySpace::Unified && axom::isMemorySpaceAvailable(MemorySpace::Unified)); +#else + return m == MemorySpace::Dynamic || m == MemorySpace::Malloc || m == MemorySpace::Host; #endif - ; } static bool usesAllocId(int allocId) noexcept { diff --git a/src/axom/core/memory_management.cpp b/src/axom/core/memory_management.cpp index 9aa818b449..cf8507acd1 100644 --- a/src/axom/core/memory_management.cpp +++ b/src/axom/core/memory_management.cpp @@ -15,9 +15,228 @@ #endif #endif +#include + namespace axom { +namespace +{ +const char* memorySpaceName(MemorySpace space) noexcept +{ + switch(space) + { + case MemorySpace::Malloc: + return "Malloc"; + case MemorySpace::Dynamic: + return "Dynamic"; + case MemorySpace::Host: + return "Host"; +#if defined(AXOM_USE_UMPIRE) + case MemorySpace::Device: + return "Device"; + case MemorySpace::Unified: + return "Unified"; + case MemorySpace::Pinned: + return "Pinned"; + case MemorySpace::Constant: + return "Constant"; +#endif + } + + return "Unknown"; +} + +int platformHostAllocatorID() noexcept +{ +#if defined(AXOM_USE_UMPIRE) + return getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); +#else + return MALLOC_ALLOCATOR_ID; +#endif +} + +int initialDefaultHostAllocatorID() noexcept { return MALLOC_ALLOCATOR_ID; } + +struct HostAllocatorConfig +{ + explicit HostAllocatorConfig(int allocId) noexcept : allocatorId {allocId}, locked {false} { } + + int get() const noexcept { return allocatorId.load(std::memory_order_relaxed); } + + void set(int allocId) noexcept { allocatorId.store(allocId, std::memory_order_relaxed); } + + bool isLocked() const noexcept { return locked.load(std::memory_order_relaxed); } + + void markUsed() noexcept { locked.store(true, std::memory_order_relaxed); } + +private: + std::atomic allocatorId; + std::atomic locked; +}; + +HostAllocatorConfig& defaultHostAllocatorConfig() noexcept +{ + static HostAllocatorConfig config {initialDefaultHostAllocatorID()}; + return config; +} +} // namespace + +namespace detail +{ + +void markDefaultHostAllocatorUsed(int allocId) noexcept +{ + const int defaultHostAllocId = defaultHostAllocatorConfig().get(); + if(defaultHostAllocId != MALLOC_ALLOCATOR_ID && allocId == defaultHostAllocId) + { + defaultHostAllocatorConfig().markUsed(); + } +} + +} // namespace detail + +bool isMemorySpaceAvailable(MemorySpace space) noexcept +{ + switch(space) + { + case MemorySpace::Malloc: + case MemorySpace::Dynamic: + case MemorySpace::Host: + return true; +#if defined(AXOM_USE_UMPIRE) + + case MemorySpace::Device: + #if defined(UMPIRE_ENABLE_DEVICE) + return true; + #else + return false; + #endif + case MemorySpace::Unified: + #if defined(UMPIRE_ENABLE_UM) + return true; + #else + return false; + #endif + case MemorySpace::Pinned: + #if defined(UMPIRE_ENABLE_PINNED) + return true; + #else + return false; + #endif + case MemorySpace::Constant: + #if defined(UMPIRE_ENABLE_CONST) + return true; + #else + return false; + #endif + +#endif + } + + return false; +} + +int getAllocatorIDFromMemorySpace(MemorySpace space) +{ + switch(space) + { + case MemorySpace::Dynamic: + return getDefaultAllocatorID(); + case MemorySpace::Malloc: + return MALLOC_ALLOCATOR_ID; + case MemorySpace::Host: + return getDefaultHostAllocatorID(); + +#if defined(AXOM_USE_UMPIRE) + case MemorySpace::Device: + #if defined(UMPIRE_ENABLE_DEVICE) + return getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Device); + #else + break; + #endif + case MemorySpace::Unified: + #if defined(UMPIRE_ENABLE_UM) + return getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Unified); + #else + break; + #endif + case MemorySpace::Pinned: + #if defined(UMPIRE_ENABLE_PINNED) + return getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Pinned); + #else + break; + #endif + case MemorySpace::Constant: + #if defined(UMPIRE_ENABLE_CONST) + return getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Constant); + #else + break; + #endif + +#endif + } + + std::cerr << "Axom memory space \"" << memorySpaceName(space) + << "\" is not available in this build." << std::endl; + axom::utilities::processAbort(); + + return INVALID_ALLOCATOR_ID; // Silence warning. +} + +void setDefaultAllocator(MemorySpace space) +{ +#if defined(AXOM_USE_UMPIRE) + if(space == MemorySpace::Host) + { + setDefaultAllocator(platformHostAllocatorID()); + return; + } +#endif + setDefaultAllocator(getAllocatorIDFromMemorySpace(space)); +} + +void setDefaultHostAllocator(MemorySpace space) +{ + switch(space) + { + case MemorySpace::Malloc: + setDefaultHostAllocator(MALLOC_ALLOCATOR_ID); + return; + case MemorySpace::Host: + setDefaultHostAllocator(platformHostAllocatorID()); + return; + default: + break; + } + + std::cerr << "Axom memory space \"" << memorySpaceName(space) + << "\" is not a valid default host allocator." << std::endl; + axom::utilities::processAbort(); +} + +void setDefaultHostAllocator(int allocId) +{ + if(!isAllocatorCompatibleWithMemorySpace(allocId, MemorySpace::Host)) + { + std::cerr << "Allocator id " << allocId << " is not compatible with Axom's host memory space." + << std::endl; + axom::utilities::processAbort(); + } + + const int currentAllocId = defaultHostAllocatorConfig().get(); + if(currentAllocId != allocId && defaultHostAllocatorConfig().isLocked()) + { + std::cerr << "Default host allocator cannot be changed from " << currentAllocId << " to " + << allocId << " after an allocation has been made from it." << std::endl; + axom::utilities::processAbort(); + } + + defaultHostAllocatorConfig().set(allocId); +} + +int getDefaultHostAllocatorID() { return defaultHostAllocatorConfig().get(); } + bool isSharedMemoryAllocator(int allocID) { bool isShared = false; diff --git a/src/axom/core/memory_management.hpp b/src/axom/core/memory_management.hpp index 5f6fd8bd41..603387ceb6 100644 --- a/src/axom/core/memory_management.hpp +++ b/src/axom/core/memory_management.hpp @@ -12,7 +12,7 @@ #include "axom/core/utilities/Utilities.hpp" // Umpire includes -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) #include "umpire/config.hpp" #include "umpire/ResourceManager.hpp" #include "umpire/op/MemoryOperationRegistry.hpp" @@ -32,7 +32,7 @@ namespace axom { -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) namespace detail { /*! @@ -73,6 +73,8 @@ inline const UmpireCopyContext& getUmpireCopyContext() noexcept constexpr int INVALID_ALLOCATOR_ID = -1; //!< Place holder for no/unknown allocator constexpr int MALLOC_ALLOCATOR_ID = -3; //!< Refers to MemorySpace::Malloc +struct HostAllocator; + /*! * \brief Returns whether \a allocatorId is a valid Axom allocator id. * @@ -114,13 +116,13 @@ inline bool isValidAllocatorID(int allocatorId) noexcept enum class MemorySpace { Malloc, //!< Host memory using malloc, free and realloc - Dynamic, //!< Refers to Umpire's current default allocator -#ifdef AXOM_USE_UMPIRE - Host, //!< Umpire's host memory space - Device, //!< Umpire's device memory space - Unified, //!< Umpire's unified memory space - Pinned, //!< Umpire's pinned memory space - Constant //!< Umpire's constant memory space + Dynamic, //!< Default template param value for Axom array types (see above) + Host, //!< Default host memory space (i.e., CPU-accessible) +#if defined(AXOM_USE_UMPIRE) + Device, //!< Device memory space + Unified, //!< Unified memory space + Pinned, //!< Pinned host memory space + Constant //!< Constant device memory space #endif }; // _memory_space_end @@ -129,7 +131,57 @@ enum class MemorySpace /// \name Memory Management Routines /// @{ -#ifdef AXOM_USE_UMPIRE +/*! + * \brief Returns true if memory space is available in the current build. + * + * \note `MemorySpace::Malloc`, `MemorySpace::Dynamic`, and `MemorySpace::Host` + * are always available. The remaining spaces require Umpire support for + * the corresponding resource. + */ +bool isMemorySpaceAvailable(MemorySpace space) noexcept; + +/*! + * \brief Returns the allocator ID corresponding to a memory space. + * + * \note `MemorySpace::Dynamic` resolves to the current default allocator. + * \note `MemorySpace::Host` resolves to Axom's current default host allocator. + * This is a legacy compatibility path; new APIs that allocate + * host-resident storage or host staging should pass `HostAllocator` + * explicitly. + * \note This function aborts if the requested memory space is unavailable in + * the current build. + */ +int getAllocatorIDFromMemorySpace(MemorySpace space); + +/*! + * \brief Sets the default memory allocator using an Axom MemorySpace enum value. + * + * \note When Axom is built without Umpire, setting the default allocator has + * no effect and host-backed memory spaces resolve to malloc. + * \note In Umpire builds, `MemorySpace::Host` selects Umpire's Host allocator. + * Use `setDefaultHostAllocator()` to configure Axom's host-only + * allocation path. + */ +void setDefaultAllocator(MemorySpace space); + +/*! + * \brief Sets the default host allocator using Axom MemorySpace enum value. + * + * \note `MemorySpace::Malloc` selects Axom's malloc-backed host allocator. + * \note `MemorySpace::Host` resets to the platform host allocator + * (Umpire Host when available). + * \note Only `MemorySpace::Malloc` and `MemorySpace::Host` are accepted. + */ +void setDefaultHostAllocator(MemorySpace space); + +/*! + * \brief Sets the default host allocator using an allocator ID. + * + * \note Accepted allocator IDs must be compatible with `MemorySpace::Host`. + */ +void setDefaultHostAllocator(int allocId); + +#if defined(AXOM_USE_UMPIRE) /*! * \brief Returns the ID of the predefined allocator for a given resource. @@ -164,7 +216,15 @@ inline void setDefaultAllocator(umpire::resource::MemoryResourceType resource_ty */ inline void setDefaultAllocator(int allocId) { -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) + if(allocId == MALLOC_ALLOCATOR_ID) + { + std::cerr << "Cannot set Axom's malloc allocator as the global default " + "allocator when Umpire is enabled. Use setDefaultHostAllocator() " + "to configure host-side allocations." + << std::endl; + axom::utilities::processAbort(); + } umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); umpire::Allocator allocator = rm.getAllocator(allocId); rm.setDefaultAllocator(allocator); @@ -181,13 +241,35 @@ inline void setDefaultAllocator(int allocId) */ inline int getDefaultAllocatorID() { -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) return umpire::ResourceManager::getInstance().getDefaultAllocator().getId(); #else return MALLOC_ALLOCATOR_ID; #endif } +/*! + * \brief Returns the ID of the current default host allocator. + * + * \return The current default host allocator ID. This is initialized to + * `MALLOC_ALLOCATOR_ID` in all builds. + * + * \note This is a process-global default used by legacy convenience paths that + * resolve `MemorySpace::Host` through global state. Prefer passing an + * explicit `HostAllocator` to APIs that allocate host-resident storage + * or host staging/scratch memory. + */ +int getDefaultHostAllocatorID(); + +/*! + * \brief Returns whether an allocator ID is compatible with a memory space. + * + * \note `MemorySpace::Host` accepts both host allocators and + * `MALLOC_ALLOCATOR_ID`. + * \note `MemorySpace::Dynamic` accepts any valid allocator ID. + */ +bool isAllocatorCompatibleWithMemorySpace(int allocId, MemorySpace space) noexcept; + /*! * \brief Get the allocator id from which data has been allocated. * \return Allocator id. If Umpire doesn't have an allocator for the @@ -199,7 +281,7 @@ inline int getDefaultAllocatorID() */ inline int getAllocatorIDFromPointer(const void* ptr) { -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); if(rm.hasAllocator(const_cast(ptr))) { @@ -295,7 +377,8 @@ inline void deallocate(T*& p) noexcept; * current allocator's memory space. This follows the semantics of * Umpire's reallocate function. * \note When p is a null pointer, allocID is used to allocate the data. - * Otherwise, it is unused. + * In Umpire builds, passing a different valid allocator can migrate a + * non-null allocation to that allocator. */ template inline T* reallocate(T* p, std::size_t n, int allocID = getDefaultAllocatorID()) noexcept; @@ -320,6 +403,11 @@ inline void copy(void* dst, const void* src, std::size_t numbytes) noexcept; * \param [in] n the number of items to copy. * \param [in] The value to copy. It must be trivially copyable for use with GPU. * + * \note This compatibility overload uses Axom's current default host allocator + * for any host scratch needed to fill non-host allocations. Prefer the + * overload that accepts `HostAllocator` when host allocator ownership is + * available. + * * \note When using Umpire if dst is not registered with the * ResourceManager then the default host allocation strategy is assumed for * that pointer. @@ -327,6 +415,22 @@ inline void copy(void* dst, const void* src, std::size_t numbytes) noexcept; template inline void fill(void* dst, std::size_t n, const T& value) noexcept; +/*! + * \brief Fills memory with a value, using an explicit host allocator for any + * host-resident scratch needed to fill non-host allocations. + * + * \param [in/out] dst the destination to copy to. + * \param [in] n the number of items to copy. + * \param [in] value the value to copy. It must be trivially copyable for use with GPU. + * \param [in] hostAllocator allocator used for host-resident scratch. + * + * \note When using Umpire if dst is not registered with the + * ResourceManager then the default host allocation strategy is assumed for + * that pointer. + */ +template +inline void fill(void* dst, std::size_t n, const T& value, HostAllocator hostAllocator) noexcept; + /// @} // _memory_management_routines_end @@ -351,26 +455,210 @@ struct Allocator int m_id; }; +/*! + * \brief Wrapper type representing an allocator ID that is valid for host allocations. + * + * This type is intended for APIs where the allocator specifically refers to host-resident + * storage or host staging memory. + * + * Default construction is provided for compatibility with existing defaulting + * APIs. New production code should prefer constructing `HostAllocator` from an + * explicit allocator ID and passing it through APIs that allocate host scratch. + */ +struct HostAllocator +{ +public: + AXOM_HOST_DEVICE explicit HostAllocator(int alloc_id = defaultID()) : m_id {alloc_id} + { +#if !defined(AXOM_DEVICE_CODE) + if(!axom::isAllocatorCompatibleWithMemorySpace(m_id, MemorySpace::Host)) + { + std::cerr << "Allocator id " << m_id << " is not compatible with Axom's host memory space." + << std::endl; + axom::utilities::processAbort(); + } +#endif + } + + /// \brief Returns the allocator ID. + AXOM_HOST_DEVICE int getID() const { return m_id; } + + /// \brief Returns the MemorySpace type for the given allocator. + MemorySpace getSpace() const; + +private: + AXOM_HOST_DEVICE static int defaultID() + { +#if defined(AXOM_DEVICE_CODE) + return axom::MALLOC_ALLOCATOR_ID; +#else + return axom::getDefaultHostAllocatorID(); +#endif + } + + int m_id; +}; + //------------------------------------------------------------------------------ // IMPLEMENTATION //------------------------------------------------------------------------------ +namespace detail +{ + +/// Record that default host allocator has been used to allocate memory. +void markDefaultHostAllocatorUsed(int allocId) noexcept; + +/// Enumeration used internally to make default allocator more easily identifiable. +enum class AllocationBackend +{ + Malloc, +#if defined(AXOM_USE_UMPIRE) + Umpire, +#endif + Invalid +}; + +/// Get enum value for allocator backend associated with given allocator ID +inline AllocationBackend getAllocatorBackend(int allocID) noexcept +{ + if(allocID == MALLOC_ALLOCATOR_ID) + { + return AllocationBackend::Malloc; + } + +#if defined(AXOM_USE_UMPIRE) + if(umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); rm.isAllocator(allocID)) + { + return AllocationBackend::Umpire; + } +#endif + + return AllocationBackend::Invalid; +} + +/// Get enum value for allocator backend associated with given pointer +template +inline AllocationBackend getPointerBackend(T* pointer) noexcept +{ +#if defined(AXOM_USE_UMPIRE) + if(umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); rm.hasAllocator(pointer)) + { + return AllocationBackend::Umpire; + } +#else + AXOM_UNUSED_VAR(pointer); +#endif + + return AllocationBackend::Malloc; +} + +/// Utility to abort when attempt is made to use invalid allocator ID +inline void abortOnInvalidAllocatorID(int allocID) +{ +#if defined(AXOM_USE_UMPIRE) + std::cerr << "Unrecognized allocator id " << allocID << std::endl; +#else + std::cerr + << "*** Unrecognized allocator id " << allocID + << ". Axom was NOT built with Umpire, so the only valid allocator id is MALLOC_ALLOCATOR_ID (" + << MALLOC_ALLOCATOR_ID << ")." << std::endl; +#endif + axom::utilities::processAbort(); +} + +/// Utility to abort when attempt is made to reallocate with invalid allocator ID +inline void abortOnInvalidReallocateState() +{ + std::cerr << "Unexpected allocator backend state in axom::reallocate()." << std::endl; + axom::utilities::processAbort(); +} + +/// Utility to abort when attempt is made to reallocate from malloc to Umpire or vice-versa. +inline void abortOnCrossBackendReallocate(int srcAllocID, int dstAllocID) +{ + std::cerr << "Cannot reallocate across allocator backends. Source allocator id is " << srcAllocID + << " and destination allocator id is " << dstAllocID + << ". Reallocation is only supported within malloc-backed memory or within " + "Umpire-backed memory." + << std::endl; + axom::utilities::processAbort(); +} + +/// Normalize pointer when request to reallocate to size zero is made +template +inline T* normalizeZeroSizeReallocateResult(T* pointer, std::size_t n, int allocID) noexcept +{ + if(n == 0 && pointer == nullptr) + { + return axom::allocate(0, allocID); + } + + return pointer; +} + +/// Reallocate an Axom malloc allocation +template +inline T* reallocateWithinMalloc(T* pointer, std::size_t numbytes) noexcept +{ + if(numbytes == 0) + { + axom::deallocate(pointer); + return nullptr; + } + + T* reallocated = static_cast(std::realloc(pointer, numbytes)); + return reallocated; +} + +#if defined(AXOM_USE_UMPIRE) +/// Reallocate an Umpire allocation +template +inline T* reallocateWithinUmpire(T* pointer, std::size_t numbytes) noexcept +{ + umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); + return static_cast(rm.reallocate(pointer, numbytes)); +} +#endif + +} // namespace detail + +/*! + * \brief Allocate memory using given allocator ID. + * + * \param [in] n number of bytes to allocate + * \param [in] allocID id of allocator + * \return pointer to allocated memory + * + * \note This function aborts if the allocator ID is not recognized. + */ template inline T* allocate(std::size_t n, int allocID) noexcept { const std::size_t numbytes = n * sizeof(T); -#ifdef AXOM_USE_UMPIRE - if(umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); rm.isAllocator(allocID)) +#if defined(AXOM_USE_UMPIRE) + umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); + if(rm.isAllocator(allocID)) { umpire::Allocator allocator = rm.getAllocator(allocID); - return static_cast(allocator.allocate(numbytes)); + T* pointer = static_cast(allocator.allocate(numbytes)); + if(pointer != nullptr) + { + detail::markDefaultHostAllocatorUsed(allocID); + } + return pointer; } #endif if(allocID == MALLOC_ALLOCATOR_ID) { - return static_cast(std::malloc(numbytes)); + T* pointer = static_cast(std::malloc(numbytes)); + if(pointer != nullptr) + { + detail::markDefaultHostAllocatorUsed(allocID); + } + return pointer; } std::cerr << "Unrecognized allocator id " << allocID << std::endl; @@ -379,24 +667,46 @@ inline T* allocate(std::size_t n, int allocID) noexcept return nullptr; // Silence warning. } +/*! + * \brief Allocate 'named' memory using given allocator ID. + * + * \param [in] n number of bytes to allocate + * \param [in] name name of allocation + * \param [in] allocID id of allocator + * \return pointer to allocated memory + * + * \note Name is used only in case of Umpire allocation + * \note This function aborts if the allocator ID is not recognized. + */ template inline T* allocate(std::size_t n, const std::string& name, int allocID) noexcept { const std::size_t numbytes = n * sizeof(T); -#ifdef AXOM_USE_UMPIRE - if(umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); rm.isAllocator(allocID)) +#if defined(AXOM_USE_UMPIRE) + umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); + if(rm.isAllocator(allocID)) { umpire::Allocator allocator = rm.getAllocator(allocID); - return name.empty() ? static_cast(allocator.allocate(numbytes)) - : static_cast(allocator.allocate(name, numbytes)); + T* pointer = name.empty() ? static_cast(allocator.allocate(numbytes)) + : static_cast(allocator.allocate(name, numbytes)); + if(pointer != nullptr) + { + detail::markDefaultHostAllocatorUsed(allocID); + } + return pointer; } #endif if(allocID == MALLOC_ALLOCATOR_ID) { AXOM_UNUSED_VAR(name); - return static_cast(std::malloc(numbytes)); + T* pointer = static_cast(std::malloc(numbytes)); + if(pointer != nullptr) + { + detail::markDefaultHostAllocatorUsed(allocID); + } + return pointer; } std::cerr << "Unrecognized allocator id " << allocID << std::endl; @@ -404,7 +714,10 @@ inline T* allocate(std::size_t n, const std::string& name, int allocID) noexcept return nullptr; // Silence warning. } -//------------------------------------------------------------------------------ + +/*! + * \brief Deallocate memory referenced by given pointer + */ template inline void deallocate(T*& pointer) noexcept { @@ -413,7 +726,7 @@ inline void deallocate(T*& pointer) noexcept return; } -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); if(rm.hasAllocator(pointer)) @@ -429,76 +742,72 @@ inline void deallocate(T*& pointer) noexcept pointer = nullptr; } -//------------------------------------------------------------------------------ +/*! + * \brief Reallocate memory using given allocator ID. + * + * \param [in] pointer address of memory to reallocate + * \param [in] n number of bytes in new allocation + * \param [in] allocID id of allocator + * \return pointer to reallocated memory + * + * \note This function aborts if the allocator ID is not recognized, + * or if attempt is made to reallocate memory allocated with + * Axom malloc using an Umpire allocator or vice versa. + */ template inline T* reallocate(T* pointer, std::size_t n, int allocID) noexcept { assert(allocID != INVALID_ALLOCATOR_ID); const std::size_t numbytes = n * sizeof(T); + const detail::AllocationBackend dst = detail::getAllocatorBackend(allocID); -#if defined(AXOM_USE_UMPIRE) + if(dst == detail::AllocationBackend::Invalid) + { + detail::abortOnInvalidAllocatorID(allocID); + } - umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); - if(rm.isAllocator(allocID)) + if(pointer == nullptr) { - if(pointer == nullptr) + return detail::normalizeZeroSizeReallocateResult(axom::allocate(n, allocID), n, allocID); + } + + const detail::AllocationBackend src = detail::getPointerBackend(pointer); + + if(src == dst) + { + if(src == detail::AllocationBackend::Malloc) + { + pointer = detail::reallocateWithinMalloc(pointer, numbytes); + } +#if defined(AXOM_USE_UMPIRE) + else if(src == detail::AllocationBackend::Umpire) { - pointer = axom::allocate(n, allocID); + pointer = detail::reallocateWithinUmpire(pointer, numbytes); } +#endif else { - if(rm.hasAllocator(pointer)) - { - pointer = static_cast(rm.reallocate(pointer, numbytes)); - } - else - { - /* - * Reallocate from non-Umpire to Umpire, manually, using - * allocate, copy and deallocate. Because we don't know the - * current size, we first do a (extra) reallocate within the - * current space just so we have the size for the copy. - * Is there a better way? - */ - auto tmpPointer = std::realloc(pointer, numbytes); - pointer = axom::allocate(n, allocID); - copy(pointer, tmpPointer, numbytes); - deallocate(tmpPointer); - } + detail::abortOnInvalidReallocateState(); } - return pointer; - } - -#else - if(allocID == MALLOC_ALLOCATOR_ID) - { - pointer = static_cast(std::realloc(pointer, numbytes)); + return detail::normalizeZeroSizeReallocateResult(pointer, n, allocID); } - else - { - std::cerr << "*** Unrecognized allocator id " - << allocID << ". Axom was NOT built with Umpire, so the only valid allocator id is MALLOC_ALLOCATOR_ID (" - << MALLOC_ALLOCATOR_ID << ")." << std::endl; - axom::utilities::processAbort(); - } - - // Consistently handle realloc(0) for std::realloc to match Umpire's behavior - if(n == 0 && pointer == nullptr) - { - pointer = axom::allocate(0); - } - -#endif - return pointer; + detail::abortOnCrossBackendReallocate(getAllocatorIDFromPointer(pointer), allocID); + return nullptr; // Silence warning. } -//------------------------------------------------------------------------------ +/*! + * \brief Copy given number of bytes from one memory chunk to another + * + * \param [in] dst copy destination + * \param [in] src copy src + * \param [in] numbytes number of bytes to copy + */ inline void copy(void* dst, const void* src, std::size_t numbytes) noexcept { -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) const auto& copyContext = detail::getUmpireCopyContext(); umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); umpire::op::MemoryOperationRegistry& op_registry = *copyContext.operationRegistry; @@ -529,12 +838,33 @@ inline void copy(void* dst, const void* src, std::size_t numbytes) noexcept #endif } -//------------------------------------------------------------------------------ +/*! + * \brief Fill allocation with given value. + * + * \param [in] dst fill destination + * \param [in] n number of values (of type T) to fill + * \param [in] value value to fill + */ template inline void fill(void* dst, std::size_t n, const T& value) noexcept +{ + axom::fill(dst, n, value, HostAllocator {}); +} + +/*! + * \brief Fill allocation with given value. + * + * \param [in] dst fill destination + * \param [in] n number of values (of type T) to fill + * \param [in] value value to fill + * \param [in] hostAllocator allocator used for host-resident scratch + */ +template +inline void fill(void* dst, std::size_t n, const T& value, HostAllocator hostAllocator) noexcept { bool doHostFill = true; -#ifdef AXOM_USE_UMPIRE + AXOM_UNUSED_VAR(hostAllocator); +#if defined(AXOM_USE_UMPIRE) // Since data might be copied to GPU, it needs to be trivially copyable. static_assert(std::is_trivially_copyable::value, "value must be trivially copyable."); auto& rm = umpire::ResourceManager::getInstance(); @@ -548,12 +878,12 @@ inline void fill(void* dst, std::size_t n, const T& value) noexcept // Device memory: fill on host, then copy to device const auto num_bytes = n * sizeof(T); - T* src = allocate(num_bytes, rm.getDefaultAllocator().getId()); + T* src = allocate(n, hostAllocator.getID()); for(std::size_t i = 0; i < n; ++i) { src[i] = value; } - rm.copy(dst, src, num_bytes); + axom::copy(dst, src, num_bytes); deallocate(src); } } @@ -600,7 +930,7 @@ inline int getAllocatorID() */ inline MemorySpace getAllocatorSpace(int allocatorId) { -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) using ump_res_type = typename umpire::MemoryResourceTraits::resource_type; umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); @@ -636,42 +966,88 @@ inline MemorySpace getAllocatorSpace(int allocatorId) return MemorySpace::Malloc; // Silence warning. } -#ifdef AXOM_USE_UMPIRE - template <> inline int getAllocatorID() { - return axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); + // Legacy convenience: resolves MemorySpace::Host via the process-global + // default host allocator. Prefer passing an explicit HostAllocator. + return axom::getAllocatorIDFromMemorySpace(MemorySpace::Host); } +#if defined(AXOM_USE_UMPIRE) template <> inline int getAllocatorID() { - return axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Device); + return axom::getAllocatorIDFromMemorySpace(MemorySpace::Device); } template <> inline int getAllocatorID() { - return axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Unified); + return axom::getAllocatorIDFromMemorySpace(MemorySpace::Unified); } template <> inline int getAllocatorID() { - return axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Pinned); + return axom::getAllocatorIDFromMemorySpace(MemorySpace::Pinned); } template <> inline int getAllocatorID() { - return axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Constant); + return axom::getAllocatorIDFromMemorySpace(MemorySpace::Constant); } - #endif } // namespace detail +/*! + * \brief Returns true if memory allocated by allocator with given ID + * is accessible on host; else return false. + */ +inline bool isHostAccessibleAllocatorID(int allocId) +{ + switch(detail::getAllocatorSpace(allocId)) + { + case axom::MemorySpace::Malloc: + case axom::MemorySpace::Host: +#if defined(AXOM_USE_UMPIRE) + case axom::MemorySpace::Unified: + case axom::MemorySpace::Pinned: +#endif + return true; + default: + return false; + } +} + +/*! + * \brief Returns true if allocator with given ID is compatible with given + * memory space enum value; else return false. + */ +inline bool isAllocatorCompatibleWithMemorySpace(int allocId, MemorySpace space) noexcept +{ + if(!isValidAllocatorID(allocId)) + { + return false; + } + + if(space == MemorySpace::Dynamic) + { + return true; + } + + const auto allocSpace = detail::getAllocatorSpace(allocId); + + if(space == MemorySpace::Host) + { + return allocSpace == MemorySpace::Host || allocSpace == MemorySpace::Malloc; + } + + return allocSpace == space; +} + /*! * \brief Determines whether an allocator id is on device. * @@ -690,4 +1066,6 @@ inline bool isDeviceAllocator(int AXOM_UNUSED_PARAM(allocator_id)) { return fals inline MemorySpace Allocator::getSpace() const { return axom::detail::getAllocatorSpace(m_id); } +inline MemorySpace HostAllocator::getSpace() const { return axom::detail::getAllocatorSpace(m_id); } + } // namespace axom diff --git a/src/axom/core/tests/core_array.hpp b/src/axom/core/tests/core_array.hpp index 19276b69ed..21bc6555c8 100644 --- a/src/axom/core/tests/core_array.hpp +++ b/src/axom/core/tests/core_array.hpp @@ -10,13 +10,17 @@ #include "axom/core/ArrayView.hpp" #include "axom/core/memory_management.hpp" #include "axom/core/execution/for_all.hpp" +#include "axom/core/utilities/MemoryTesting.hpp" #include "gtest/gtest.h" #include +#include +#include namespace { + /*! * \brief Calculate the new capacity for an Array given an increase in the size. * \param [in] v, the Array in question. @@ -889,6 +893,8 @@ template void check_device(axom::Array& v) { const axom::IndexType size = v.size(); + const int explicit_host_alloc = + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); // Then assign to it via a raw device pointer assign_raw<<<1, 1>>>(v.data(), size); @@ -903,7 +909,7 @@ void check_device(axom::Array& v) } // Then check the contents by assigning to an explicitly Host Array - axom::Array check_raw_array_host = v; + axom::Array check_raw_array_host(v, explicit_host_alloc); EXPECT_EQ(check_raw_array_host.size(), size); for(int i = 0; i < check_raw_array_host.size(); i++) { @@ -924,7 +930,7 @@ void check_device(axom::Array& v) } // Then check the contents by assigning to an explicitly Host array - axom::Array check_view_array_host = view; + axom::Array check_view_array_host(view, explicit_host_alloc); EXPECT_EQ(check_view_array_host.size(), size); for(int i = 0; i < check_view_array_host.size(); i++) { @@ -966,6 +972,8 @@ void check_device_2D(axom::Array& v) const axom::IndexType size = v.size(); const axom::IndexType M = v.shape()[0]; const axom::IndexType N = v.shape()[1]; + const int explicit_host_alloc = + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); // Then assign to it via a raw device pointer assign_raw_2d<<<1, 1>>>(v.data(), M, N); @@ -985,7 +993,7 @@ void check_device_2D(axom::Array& v) } // Then check the contents by assigning to an explicitly Host array - axom::Array check_raw_array_host = v; + axom::Array check_raw_array_host(v, explicit_host_alloc); EXPECT_EQ(check_raw_array_host.size(), size); EXPECT_EQ(check_raw_array_host.shape(), v.shape()); @@ -1016,7 +1024,7 @@ void check_device_2D(axom::Array& v) } // Then check the contents by assigning to an explicitly Host array - axom::Array check_view_array_host = view; + axom::Array check_view_array_host(view, explicit_host_alloc); EXPECT_EQ(check_view_array_host.size(), size); EXPECT_EQ(check_view_array_host.shape(), v.shape()); @@ -1097,6 +1105,11 @@ TEST(core_array, checkFill) #if defined(AXOM_USE_GPU) && defined(AXOM_GPUCC) && defined(AXOM_USE_UMPIRE) TEST(core_array, checkFillDevice) { + if(!axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Device)) + { + GTEST_SKIP() << "Device allocator is unavailable at runtime."; + } + for(axom::IndexType capacity = 2; capacity < 512; capacity *= 2) { axom::IndexType size = capacity / 2; @@ -1150,6 +1163,11 @@ TEST(core_array, checkAssignView) #if defined(AXOM_USE_GPU) && defined(AXOM_GPUCC) && defined(AXOM_USE_UMPIRE) TEST(core_array, checkAssignDevice) { + if(!axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Device)) + { + GTEST_SKIP() << "Device allocator is unavailable at runtime."; + } + // Check Array::assign methods when using device memory. const axom::IndexType size = 100, capacity = 100; axom::Array v_int(size, capacity); @@ -1161,6 +1179,11 @@ TEST(core_array, checkAssignDevice) TEST(core_array, checkAssignViewDevice) { + if(!axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Device)) + { + GTEST_SKIP() << "Device allocator is unavailable at runtime."; + } + // Check Array::assign methods when using device memory. const axom::IndexType size = 100, capacity = 100; axom::Array v_int(size, capacity); @@ -1326,25 +1349,34 @@ TEST(core_array, checkAlloc) std::vector memory_locations { #if defined(AXOM_USE_UMPIRE) axom::getUmpireResourceAllocatorID(umpire::resource::Host) - #if defined(UMPIRE_ENABLE_DEVICE) - , - axom::getUmpireResourceAllocatorID(umpire::resource::Device) - #endif - #if defined(UMPIRE_ENABLE_UM) - , - axom::getUmpireResourceAllocatorID(umpire::resource::Unified) - #endif - #if defined(UMPIRE_ENABLE_CONST) - , - axom::getUmpireResourceAllocatorID(umpire::resource::Constant) - #endif - #if defined(UMPIRE_ENABLE_PINNED) - , - axom::getUmpireResourceAllocatorID(umpire::resource::Pinned) - #endif #endif }; +#if defined(AXOM_USE_UMPIRE) && defined(UMPIRE_ENABLE_DEVICE) + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Device)) + { + memory_locations.push_back(axom::getUmpireResourceAllocatorID(umpire::resource::Device)); + } +#endif +#if defined(AXOM_USE_UMPIRE) && defined(UMPIRE_ENABLE_UM) + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Unified)) + { + memory_locations.push_back(axom::getUmpireResourceAllocatorID(umpire::resource::Unified)); + } +#endif +#if defined(AXOM_USE_UMPIRE) && defined(UMPIRE_ENABLE_CONST) + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Constant)) + { + memory_locations.push_back(axom::getUmpireResourceAllocatorID(umpire::resource::Constant)); + } +#endif +#if defined(AXOM_USE_UMPIRE) && defined(UMPIRE_ENABLE_PINNED) + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Pinned)) + { + memory_locations.push_back(axom::getUmpireResourceAllocatorID(umpire::resource::Pinned)); + } +#endif + for(double ratio = 1.0; ratio <= 2.0; ratio += 0.5) { for(axom::IndexType capacity = 4; capacity <= 512; capacity *= 2) @@ -1359,30 +1391,43 @@ TEST(core_array, checkAlloc) ::check_alloc(v_double, id); } // Then, if Umpire is available, we can use the space as an explicit template parameter -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) #ifdef UMPIRE_ENABLE_DEVICE - axom::Array v_int_device(capacity, capacity); - ::check_alloc(v_int_device, axom::getUmpireResourceAllocatorID(umpire::resource::Device)); - axom::Array v_double_device(capacity, capacity); - ::check_alloc(v_double_device, axom::getUmpireResourceAllocatorID(umpire::resource::Device)); + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Device)) + { + axom::Array v_int_device(capacity, capacity); + ::check_alloc(v_int_device, axom::getUmpireResourceAllocatorID(umpire::resource::Device)); + axom::Array v_double_device(capacity, capacity); + ::check_alloc(v_double_device, axom::getUmpireResourceAllocatorID(umpire::resource::Device)); + } #endif #ifdef UMPIRE_ENABLE_UM - axom::Array v_int_unified(capacity, capacity); - ::check_alloc(v_int_unified, axom::getUmpireResourceAllocatorID(umpire::resource::Unified)); - axom::Array v_double_unified(capacity, capacity); - ::check_alloc(v_double_unified, axom::getUmpireResourceAllocatorID(umpire::resource::Unified)); + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Unified)) + { + axom::Array v_int_unified(capacity, capacity); + ::check_alloc(v_int_unified, axom::getUmpireResourceAllocatorID(umpire::resource::Unified)); + axom::Array v_double_unified(capacity, capacity); + ::check_alloc(v_double_unified, + axom::getUmpireResourceAllocatorID(umpire::resource::Unified)); + } #endif #ifdef UMPIRE_ENABLE_CONST - axom::Array v_int_const(capacity, capacity); - ::check_alloc(v_int_const, axom::getUmpireResourceAllocatorID(umpire::resource::Constant)); - axom::Array v_double_const(capacity, capacity); - ::check_alloc(v_double_const, axom::getUmpireResourceAllocatorID(umpire::resource::Constant)); + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Constant)) + { + axom::Array v_int_const(capacity, capacity); + ::check_alloc(v_int_const, axom::getUmpireResourceAllocatorID(umpire::resource::Constant)); + axom::Array v_double_const(capacity, capacity); + ::check_alloc(v_double_const, axom::getUmpireResourceAllocatorID(umpire::resource::Constant)); + } #endif #ifdef UMPIRE_ENABLE_PINNED - axom::Array v_int_pinned(capacity, capacity); - ::check_alloc(v_int_pinned, axom::getUmpireResourceAllocatorID(umpire::resource::Pinned)); - axom::Array v_double_pinned(capacity, capacity); - ::check_alloc(v_double_pinned, axom::getUmpireResourceAllocatorID(umpire::resource::Pinned)); + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Pinned)) + { + axom::Array v_int_pinned(capacity, capacity); + ::check_alloc(v_int_pinned, axom::getUmpireResourceAllocatorID(umpire::resource::Pinned)); + axom::Array v_double_pinned(capacity, capacity); + ::check_alloc(v_double_pinned, axom::getUmpireResourceAllocatorID(umpire::resource::Pinned)); + } #endif #endif } @@ -1482,7 +1527,9 @@ TEST(core_array, checkIterator) void checkIteratorDeviceImpl() { constexpr int SIZE = 1000; - axom::Array v_int_host(SIZE); + const int explicit_host_alloc = + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); + axom::Array v_int_host(SIZE, SIZE, explicit_host_alloc); axom::Array v_int(SIZE); /* Push 0...999 elements */ @@ -1529,7 +1576,15 @@ void checkIteratorDeviceImpl() EXPECT_EQ(v_int.size(), 0); } -TEST(core_array, checkIteratorDevice) { checkIteratorDeviceImpl(); } +TEST(core_array, checkIteratorDevice) +{ + if(!axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Device)) + { + GTEST_SKIP() << "Device allocator is unavailable at runtime."; + } + + checkIteratorDeviceImpl(); +} #endif //------------------------------------------------------------------------------ @@ -2105,6 +2160,11 @@ TEST(core_array, checkDevice) GTEST_SKIP() << "CUDA or HIP is not available, skipping tests that use Array " "in device code"; #else + if(!axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Device)) + { + GTEST_SKIP() << "Device allocator is unavailable at runtime."; + } + for(axom::IndexType capacity = 2; capacity < 512; capacity *= 2) { // Allocate a Dynamic array in Device memory @@ -2140,6 +2200,11 @@ TEST(core_array, checkDevice2D) GTEST_SKIP() << "CUDA or HIP is not available, skipping tests that use Array " "in device code"; #else + if(!axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Device)) + { + GTEST_SKIP() << "Device allocator is unavailable at runtime."; + } + for(axom::IndexType capacity = 2; capacity < 512; capacity *= 2) { // Allocate an explicitly Device array @@ -2205,14 +2270,21 @@ TEST(core_array, checkDefaultInitializationDevice) GTEST_SKIP() << "CUDA or HIP is not available, skipping tests that use Array " "in device code"; #else + if(!axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Device)) + { + GTEST_SKIP() << "Device allocator is unavailable at runtime."; + } + constexpr int MAGIC_INT = 255; for(axom::IndexType capacity = 2; capacity < 512; capacity *= 2) { + const int explicit_host_alloc = + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); // Allocate an explicitly Device array of ints (zero-initialized) axom::Array v_int(capacity); // Then copy it to the host - axom::Array v_int_host(v_int); + axom::Array v_int_host(v_int, explicit_host_alloc); for(const auto ele : v_int_host) { @@ -2223,7 +2295,8 @@ TEST(core_array, checkDefaultInitializationDevice) axom::Array v_has_default_device(capacity); // Then copy it to the host - axom::Array v_has_default_host(v_has_default_device); + axom::Array v_has_default_host(v_has_default_device, + explicit_host_alloc); for(const auto& ele : v_has_default_host) { @@ -2319,6 +2392,208 @@ TEST(core_array, checkUninitialized) } } +//------------------------------------------------------------------------------ +TEST(core_array, host_space_accepts_malloc_allocator) +{ + EXPECT_EXIT(([]() { + axom::setDefaultHostAllocator(axom::MemorySpace::Malloc); + + axom::Array arr(8, 8, axom::MALLOC_ALLOCATOR_ID); + if(arr.getAllocatorID() != axom::MALLOC_ALLOCATOR_ID) + { + std::exit(1); + } + + for(int i = 0; i < arr.size(); ++i) + { + arr[i] = i; + } + + axom::ArrayView view(arr); + if(view.getAllocatorID() != axom::MALLOC_ALLOCATOR_ID || arr.size() != view.size()) + { + std::exit(1); + } + + for(int i = 0; i < view.size(); ++i) + { + if(i != view[i]) + { + std::exit(1); + } + } + + std::exit(0); + })(), + ::testing::ExitedWithCode(0), + ""); +} + +//------------------------------------------------------------------------------ +TEST(core_array, host_space_copy_preserves_malloc_allocator) +{ + EXPECT_EXIT(([]() { + axom::setDefaultHostAllocator(axom::MemorySpace::Malloc); + + axom::Array src(8, 8, axom::MALLOC_ALLOCATOR_ID); + for(int i = 0; i < src.size(); ++i) + { + src[i] = 2 * i; + } + + axom::Array dst(src); + if(dst.getAllocatorID() != axom::MALLOC_ALLOCATOR_ID || src.size() != dst.size()) + { + std::exit(1); + } + + for(int i = 0; i < dst.size(); ++i) + { + if(2 * i != dst[i]) + { + std::exit(1); + } + } + + std::exit(0); + })(), + ::testing::ExitedWithCode(0), + ""); +} + +#if defined(AXOM_USE_UMPIRE) +//------------------------------------------------------------------------------ +TEST(core_array, host_space_uses_umpire_host_allocator) +{ + EXPECT_EXIT(([]() { + const int hostAllocatorID = + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); + axom::setDefaultHostAllocator(axom::MemorySpace::Host); + + axom::Array arr(8, 8); + if(arr.getAllocatorID() != hostAllocatorID) + { + std::exit(1); + } + + for(int i = 0; i < arr.size(); ++i) + { + arr[i] = i; + } + + axom::ArrayView view(arr); + if(view.getAllocatorID() != hostAllocatorID || arr.size() != view.size()) + { + std::exit(1); + } + + for(int i = 0; i < view.size(); ++i) + { + if(i != view[i]) + { + std::exit(1); + } + } + + std::exit(0); + })(), + ::testing::ExitedWithCode(0), + ""); +} + +//------------------------------------------------------------------------------ +TEST(core_array, host_space_copy_preserves_compatible_malloc_allocator_with_umpire_host_default) +{ + EXPECT_EXIT(([]() { + axom::setDefaultHostAllocator(axom::MemorySpace::Host); + + axom::Array src(8, 8, axom::MALLOC_ALLOCATOR_ID); + for(int i = 0; i < src.size(); ++i) + { + src[i] = 2 * i; + } + + axom::Array dst(src); + if(dst.getAllocatorID() != axom::MALLOC_ALLOCATOR_ID || src.size() != dst.size()) + { + std::exit(1); + } + + for(int i = 0; i < dst.size(); ++i) + { + if(2 * i != dst[i]) + { + std::exit(1); + } + } + + std::exit(0); + })(), + ::testing::ExitedWithCode(0), + ""); +} + + #if defined(UMPIRE_ENABLE_UM) || defined(UMPIRE_ENABLE_PINNED) +//------------------------------------------------------------------------------ +TEST(core_array, host_space_copy_uses_explicit_host_allocator_for_incompatible_source) +{ + EXPECT_EXIT(([]() { + #if defined(UMPIRE_ENABLE_UM) + constexpr axom::MemorySpace sourceSpace = axom::MemorySpace::Unified; + #else + constexpr axom::MemorySpace sourceSpace = axom::MemorySpace::Pinned; + #endif + if(!axom::utilities::runtimeMemorySpaceAvailable(sourceSpace)) + { + std::exit(0); + } + + const int sourceAllocatorID = axom::getAllocatorIDFromMemorySpace(sourceSpace); + const int defaultHostAllocatorID = + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); + const axom::HostAllocator explicitHostAllocator {axom::MALLOC_ALLOCATOR_ID}; + + axom::setDefaultHostAllocator(defaultHostAllocatorID); + + axom::Array src(8, 8, sourceAllocatorID); + for(int i = 0; i < src.size(); ++i) + { + src[i] = 3 * i; + } + + axom::Array dst(src, explicitHostAllocator); + axom::Array dstWithAllocator(src, + sourceAllocatorID, + explicitHostAllocator); + + if(dst.getAllocatorID() != explicitHostAllocator.getID() || + dst.getHostAllocatorID() != explicitHostAllocator.getID()) + { + std::exit(1); + } + + if(dstWithAllocator.getAllocatorID() != explicitHostAllocator.getID() || + dstWithAllocator.getHostAllocatorID() != explicitHostAllocator.getID()) + { + std::exit(1); + } + + for(int i = 0; i < src.size(); ++i) + { + if(dst[i] != 3 * i || dstWithAllocator[i] != 3 * i) + { + std::exit(1); + } + } + + std::exit(0); + })(), + ::testing::ExitedWithCode(0), + ""); +} + #endif +#endif + //------------------------------------------------------------------------------ TEST(core_array, checkConstConversion) { @@ -2614,6 +2889,11 @@ TEST(core_array, reserve_nontrivial_reloc_2) #if defined(AXOM_USE_GPU) && defined(AXOM_USE_UMPIRE) TEST(core_array, reserve_nontrivial_reloc_um) { + if(!axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Unified)) + { + GTEST_SKIP() << "Unified allocator is unavailable at runtime."; + } + const int NUM_ELEMS = 1024; axom::Array array(NUM_ELEMS, NUM_ELEMS); diff --git a/src/axom/core/tests/core_array_for_all.hpp b/src/axom/core/tests/core_array_for_all.hpp index 3484589f9c..ce50e5f14f 100644 --- a/src/axom/core/tests/core_array_for_all.hpp +++ b/src/axom/core/tests/core_array_for_all.hpp @@ -13,12 +13,16 @@ #include "axom/core/execution/execution_space.hpp" #include "axom/core/execution/synchronize.hpp" #include "axom/core/execution/for_all.hpp" +#include "axom/core/utilities/MemoryTesting.hpp" // gtest includes #include "gtest/gtest.h" +#include + namespace testing { + template struct ArrayTestParams { @@ -54,6 +58,15 @@ class core_array_for_all : public ::testing::Test using KernelArray = axom::Array; using KernelArrayView = axom::ArrayView; + void SetUp() override + { + if(!axom::utilities::runtimeMemorySpaceAvailable(exec_space_memory)) + { + GTEST_SKIP() << "Skipping test because the allocator for the kernel memory space " + << static_cast(exec_space_memory) << " is unavailable at runtime."; + } + } + static int getKernelAllocatorID() { return axom::detail::getAllocatorID(); } }; @@ -93,6 +106,11 @@ AXOM_CUDA_TEST(core_array_for_all, capture_test) using ExecSpace = axom::CUDA_EXEC<256>; using KernelArray = axom::Array; + if(!axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Device)) + { + GTEST_SKIP() << "Device allocator is unavailable at runtime."; + } + EXPECT_DEATH_IF_SUPPORTED( { // Create an array of N items using default MemorySpace for ExecSpace @@ -1124,29 +1142,36 @@ struct DeviceInsert AXOM_TYPED_TEST(core_array_for_all, device_insert) { - using ExecSpaceType = typename TestFixture::ExecSpace; - using DynamicArrayType = typename TestFixture::template DynamicTArray; - using DynamicArrayOfArrays = typename TestFixture::template DynamicTArray; + using ExecSpace = typename TestFixture::ExecSpace; + using DynamicArray = typename TestFixture::template DynamicTArray; + using DynamicArrayOfArrays = typename TestFixture::template DynamicTArray; + int hostAllocID = axom::execution_space::allocatorID(); int kernelAllocID = TestFixture::getKernelAllocatorID(); - int umAllocID = kernelAllocID; + int umAllocID = hostAllocID; #if defined(AXOM_USE_GPU) && defined(AXOM_USE_UMPIRE) // Use unified memory for frequent movement between device operations // and value checking on host - umAllocID = axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Unified); + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Unified)) + { + umAllocID = axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Unified); + } + else if(axom::execution_space::onDevice()) + { + GTEST_SKIP() << "Unified allocator is unavailable at runtime."; + } #endif - int hostAllocID = axom::execution_space::allocatorID(); constexpr axom::IndexType N = 374; DynamicArrayOfArrays arr_container(1, 1, umAllocID); - arr_container[0] = DynamicArrayType(0, N, kernelAllocID); + arr_container[0] = DynamicArray(0, N, kernelAllocID); const auto arr_v = arr_container.view(); EXPECT_EQ(arr_container[0].size(), 0); EXPECT_EQ(arr_container[0].capacity(), N); - axom::for_all( + axom::for_all( N, AXOM_LAMBDA(axom::IndexType idx) { #if defined(AXOM_USE_OPENMP) && defined(AXOM_USE_RAJA) && !defined(AXOM_DEVICE_CODE) @@ -1167,16 +1192,16 @@ AXOM_TYPED_TEST(core_array_for_all, device_insert) }); // handles synchronization, if necessary - if(axom::execution_space::async()) + if(axom::execution_space::async()) { - axom::synchronize(); + axom::synchronize(); } EXPECT_EQ(arr_container[0].size(), N); EXPECT_EQ(arr_container[0].capacity(), N); // Copy array to host. - DynamicArrayType arr_host(arr_container[0], hostAllocID); + DynamicArray arr_host(arr_container[0], hostAllocID); // Device-side inserts may occur in any order. // Sort them before we check the inserted values. @@ -1187,7 +1212,7 @@ AXOM_TYPED_TEST(core_array_for_all, device_insert) for(int i = 0; i < N; i++) { EXPECT_EQ(arr_host[i].m_value, 3 * i + 5); - if(axom::execution_space::onDevice()) + if(axom::execution_space::onDevice()) { EXPECT_EQ(arr_host[i].m_host_or_device, INSERT_ON_DEVICE); } diff --git a/src/axom/core/tests/core_execution_space.hpp b/src/axom/core/tests/core_execution_space.hpp index 26385c8f57..91426ecd40 100644 --- a/src/axom/core/tests/core_execution_space.hpp +++ b/src/axom/core/tests/core_execution_space.hpp @@ -11,10 +11,6 @@ // spin includes #include "axom/core/execution/execution_space.hpp" -#ifdef AXOM_USE_UMPIRE - #include "umpire/Umpire.hpp" // for Umpire -#endif - #ifdef AXOM_USE_RAJA #include "RAJA/RAJA.hpp" // for RAJA #endif @@ -31,6 +27,32 @@ //------------------------------------------------------------------------------ namespace { +struct ScopedDefaultHostAllocatorStateForExecution +{ + ScopedDefaultHostAllocatorStateForExecution() : m_allocator(axom::getDefaultHostAllocatorID()) { } + + ~ScopedDefaultHostAllocatorStateForExecution() { axom::setDefaultHostAllocator(m_allocator); } + + int m_allocator; +}; + +struct ScopedDefaultAllocatorStateForExecution +{ + ScopedDefaultAllocatorStateForExecution() + : m_defaultAllocator(axom::getDefaultAllocatorID()) + , m_defaultHostAllocator(axom::getDefaultHostAllocatorID()) + { } + + ~ScopedDefaultAllocatorStateForExecution() + { + axom::setDefaultAllocator(m_defaultAllocator); + axom::setDefaultHostAllocator(m_defaultHostAllocator); + } + + int m_defaultAllocator; + int m_defaultHostAllocator; +}; + template void check_valid() { @@ -130,7 +152,7 @@ TEST(core_execution_space, check_seq_exec) constexpr bool IS_ASYNC = false; constexpr bool ON_DEVICE = false; - int allocator_id = axom::getUmpireResourceAllocatorID(umpire::resource::Host); + int allocator_id = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Host); check_execution_mappings 2022 RAJA::seq_exec, @@ -153,7 +175,7 @@ TEST(core_execution_space, check_omp_exec) constexpr bool IS_ASYNC = false; constexpr bool ON_DEVICE = false; - int allocator_id = axom::getUmpireResourceAllocatorID(umpire::resource::Host); + int allocator_id = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Host); check_execution_mappings::allocatorID()) + { + std::exit(1); + } + + #if defined(AXOM_USE_OPENMP) + if(axom::MALLOC_ALLOCATOR_ID != axom::execution_space::allocatorID()) + { + std::exit(1); + } + #endif + + std::exit(0); + })(), + ::testing::ExitedWithCode(0), + ""); +} + +TEST(core_execution_space, host_exec_uses_umpire_host_allocator) +{ + EXPECT_EXIT(([]() { + const int hostAllocatorID = + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); + + axom::setDefaultHostAllocator(axom::MemorySpace::Host); + + if(hostAllocatorID != axom::execution_space::allocatorID()) + { + std::exit(1); + } + + #if defined(AXOM_USE_OPENMP) + if(hostAllocatorID != axom::execution_space::allocatorID()) + { + std::exit(1); + } + #endif + + std::exit(0); + })(), + ::testing::ExitedWithCode(0), + ""); +} + +TEST(core_execution_space, host_exec_ignores_global_default_allocator) +{ + EXPECT_EXIT(([]() { + const int hostAllocatorID = + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); + + axom::setDefaultHostAllocator(axom::MemorySpace::Malloc); + axom::setDefaultAllocator(axom::MemorySpace::Host); + + if(hostAllocatorID != axom::getDefaultAllocatorID()) + { + std::exit(1); + } + if(axom::MALLOC_ALLOCATOR_ID != axom::execution_space::allocatorID()) + { + std::exit(1); + } + + #if defined(AXOM_USE_OPENMP) + if(axom::MALLOC_ALLOCATOR_ID != axom::execution_space::allocatorID()) + { + std::exit(1); + } + #endif + + std::exit(0); + })(), + ::testing::ExitedWithCode(0), + ""); +} + //------------------------------------------------------------------------------ #if defined(AXOM_USE_CUDA) @@ -174,7 +276,7 @@ TEST(core_execution_space, check_cuda_exec) constexpr bool IS_ASYNC = false; constexpr bool ON_DEVICE = true; - int allocator_id = axom::getUmpireResourceAllocatorID(umpire::resource::Device); + int allocator_id = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Device); check_execution_mappings, RAJA::cuda_exec, RAJA::cuda_reduce, @@ -192,7 +294,7 @@ TEST(core_execution_space, check_cuda_exec_async) constexpr bool IS_ASYNC = true; constexpr bool ON_DEVICE = true; - int allocator_id = axom::getUmpireResourceAllocatorID(umpire::resource::Device); + int allocator_id = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Device); check_execution_mappings, RAJA::cuda_exec_async, RAJA::cuda_reduce, @@ -245,7 +347,7 @@ TEST(core_execution_space, check_hip_exec) constexpr bool IS_ASYNC = false; constexpr bool ON_DEVICE = true; - int allocator_id = axom::getUmpireResourceAllocatorID(umpire::resource::Device); + int allocator_id = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Device); check_execution_mappings, RAJA::hip_exec, RAJA::hip_reduce, @@ -263,7 +365,7 @@ TEST(core_execution_space, check_hip_exec_async) constexpr bool IS_ASYNC = true; constexpr bool ON_DEVICE = true; - int allocator_id = axom::getUmpireResourceAllocatorID(umpire::resource::Device); + int allocator_id = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Device); check_execution_mappings, RAJA::hip_exec_async, RAJA::hip_reduce, diff --git a/src/axom/core/tests/core_memory_management.hpp b/src/axom/core/tests/core_memory_management.hpp index e376e8bd44..b514fd5f00 100644 --- a/src/axom/core/tests/core_memory_management.hpp +++ b/src/axom/core/tests/core_memory_management.hpp @@ -9,13 +9,19 @@ #include "gtest/gtest.h" #include "axom/core/memory_management.hpp" +#include "axom/core/utilities/MemoryTesting.hpp" -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) #include "umpire/config.hpp" #include "umpire/Allocator.hpp" #include "umpire/ResourceManager.hpp" #endif +#include +#include +#include +#include + //------------------------------------------------------------------------------ // HELPER METHODS //------------------------------------------------------------------------------ @@ -24,12 +30,39 @@ // in check_alloc_realloc_free when reallocating to 3 * ARRAY_SIZE. constexpr int ARRAY_SIZE = 5345; +struct ScopedDefaultAllocatorState +{ + ScopedDefaultAllocatorState() + : m_defaultAllocator(axom::getDefaultAllocatorID()) + , m_defaultHostAllocator(axom::getDefaultHostAllocatorID()) + { } + + ~ScopedDefaultAllocatorState() + { + axom::setDefaultAllocator(m_defaultAllocator); + axom::setDefaultHostAllocator(m_defaultHostAllocator); + } + + int m_defaultAllocator; + int m_defaultHostAllocator; +}; + +#if defined(AXOM_USE_UMPIRE) +void appendAllocatorIDIfAvailable(std::vector& allocatorIds, axom::MemorySpace space) +{ + if(axom::utilities::runtimeMemorySpaceAvailable(space)) + { + allocatorIds.push_back(axom::getAllocatorIDFromMemorySpace(space)); + } +} +#endif + class CopyTest : public ::testing::TestWithParam<::testing::tuple> { public: void SetUp() override { -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); #endif @@ -48,7 +81,7 @@ class CopyTest : public ::testing::TestWithParam<::testing::tuple(size, allocatorID); if(size > 0) { - umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); - EXPECT_EQ(allocatorID, rm.getAllocator(buffer).getId()); + EXPECT_EQ(allocatorID, axom::getAllocatorIDFromPointer(buffer)); } #else int* buffer = axom::allocate(size); @@ -164,7 +196,7 @@ void check_alloc_and_free(bool hostAccessible = true) } } -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) void check_alloc_realloc_free(int allocatorID = axom::getDefaultAllocatorID(), bool hostAccessible = true) #else @@ -175,13 +207,12 @@ void check_alloc_realloc_free(bool hostAccessible = true) { int buffer_size = size; -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) int* buffer = axom::allocate(buffer_size, allocatorID); - umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); if(buffer_size > 0) { - ASSERT_EQ(allocatorID, rm.getAllocator(buffer).getId()); + ASSERT_EQ(allocatorID, axom::getAllocatorIDFromPointer(buffer)); } #else int* buffer = axom::allocate(buffer_size); @@ -204,12 +235,14 @@ void check_alloc_realloc_free(bool hostAccessible = true) // Reallocate to a larger size. buffer_size *= 3; - buffer = axom::reallocate(buffer, buffer_size); -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) + buffer = axom::reallocate(buffer, buffer_size, allocatorID); if(buffer_size > 0) { - ASSERT_EQ(allocatorID, rm.getAllocator(buffer).getId()); + ASSERT_EQ(allocatorID, axom::getAllocatorIDFromPointer(buffer)); } +#else + buffer = axom::reallocate(buffer, buffer_size); #endif if(hostAccessible) @@ -229,12 +262,14 @@ void check_alloc_realloc_free(bool hostAccessible = true) // Reallocate to a smaller size. buffer_size /= 5; - buffer = axom::reallocate(buffer, buffer_size); -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) + buffer = axom::reallocate(buffer, buffer_size, allocatorID); if(buffer_size > 0) { - ASSERT_EQ(allocatorID, rm.getAllocator(buffer).getId()); + ASSERT_EQ(allocatorID, axom::getAllocatorIDFromPointer(buffer)); } +#else + buffer = axom::reallocate(buffer, buffer_size); #endif if(hostAccessible) @@ -256,79 +291,289 @@ void check_alloc_realloc_free(bool hostAccessible = true) // UNIT TESTS //------------------------------------------------------------------------------ -#ifdef AXOM_USE_UMPIRE +TEST(core_memory_management, memory_space_availability) +{ + EXPECT_TRUE(axom::isMemorySpaceAvailable(axom::MemorySpace::Malloc)); + EXPECT_TRUE(axom::isMemorySpaceAvailable(axom::MemorySpace::Dynamic)); + EXPECT_TRUE(axom::isMemorySpaceAvailable(axom::MemorySpace::Host)); + +#if defined(AXOM_USE_UMPIRE) + + #if defined(UMPIRE_ENABLE_DEVICE) + EXPECT_TRUE(axom::isMemorySpaceAvailable(axom::MemorySpace::Device)); + #endif + + #if defined(UMPIRE_ENABLE_UM) + EXPECT_TRUE(axom::isMemorySpaceAvailable(axom::MemorySpace::Unified)); + #endif + + #if defined(UMPIRE_ENABLE_PINNED) + EXPECT_TRUE(axom::isMemorySpaceAvailable(axom::MemorySpace::Pinned)); + #endif + + #if defined(UMPIRE_ENABLE_CONST) + EXPECT_TRUE(axom::isMemorySpaceAvailable(axom::MemorySpace::Constant)); + #endif + +#endif +} + +TEST(core_memory_management, get_allocator_id_from_memory_space) +{ + EXPECT_EQ(axom::MALLOC_ALLOCATOR_ID, + axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Malloc)); + EXPECT_EQ(axom::getDefaultAllocatorID(), + axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Dynamic)); + EXPECT_EQ(axom::getDefaultHostAllocatorID(), + axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Host)); +} + +TEST(core_memory_management, allocator_memory_space_compatibility) +{ + EXPECT_TRUE(axom::isAllocatorCompatibleWithMemorySpace(axom::MALLOC_ALLOCATOR_ID, + axom::MemorySpace::Malloc)); + EXPECT_TRUE( + axom::isAllocatorCompatibleWithMemorySpace(axom::MALLOC_ALLOCATOR_ID, axom::MemorySpace::Host)); + EXPECT_TRUE(axom::isAllocatorCompatibleWithMemorySpace(axom::MALLOC_ALLOCATOR_ID, + axom::MemorySpace::Dynamic)); + +#if defined(AXOM_USE_UMPIRE) + const int platformHostAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Host); + EXPECT_TRUE( + axom::isAllocatorCompatibleWithMemorySpace(platformHostAllocatorID, axom::MemorySpace::Host)); + EXPECT_FALSE( + axom::isAllocatorCompatibleWithMemorySpace(platformHostAllocatorID, axom::MemorySpace::Malloc)); +#endif +} + +TEST(core_memory_management, set_get_default_host_allocator) +{ + ScopedDefaultAllocatorState scopedState; + const int defaultAllocatorID = axom::getDefaultAllocatorID(); + +#if defined(AXOM_USE_UMPIRE) + const int platformHostAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Host); + EXPECT_EQ(axom::MALLOC_ALLOCATOR_ID, axom::getDefaultHostAllocatorID()); +#else + const int platformHostAllocatorID = axom::MALLOC_ALLOCATOR_ID; + EXPECT_EQ(platformHostAllocatorID, axom::getDefaultHostAllocatorID()); +#endif + + axom::setDefaultHostAllocator(axom::MemorySpace::Malloc); + EXPECT_EQ(axom::MALLOC_ALLOCATOR_ID, axom::getDefaultHostAllocatorID()); + EXPECT_EQ(axom::MALLOC_ALLOCATOR_ID, axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Host)); + EXPECT_EQ(defaultAllocatorID, axom::getDefaultAllocatorID()); + EXPECT_EQ(defaultAllocatorID, axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Dynamic)); + + axom::setDefaultHostAllocator(axom::MemorySpace::Host); + EXPECT_EQ(platformHostAllocatorID, axom::getDefaultHostAllocatorID()); + EXPECT_EQ(platformHostAllocatorID, axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Host)); + EXPECT_EQ(defaultAllocatorID, axom::getDefaultAllocatorID()); + +#if defined(AXOM_USE_UMPIRE) + axom::setDefaultHostAllocator(axom::MemorySpace::Malloc); + axom::setDefaultAllocator(axom::MemorySpace::Host); + EXPECT_EQ(platformHostAllocatorID, axom::getDefaultAllocatorID()); + EXPECT_EQ(axom::MALLOC_ALLOCATOR_ID, axom::getDefaultHostAllocatorID()); +#endif +} + +#if defined(AXOM_USE_UMPIRE) + +bool hostAllocationUsesExpectedAllocator(int selectedHostAllocId, + int expectedAllocId, + bool setGlobalDefaultToHost = false) +{ + axom::setDefaultHostAllocator(selectedHostAllocId); + + if(setGlobalDefaultToHost) + { + axom::setDefaultAllocator(axom::MemorySpace::Host); + } + + const int resolvedHostAllocatorID = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Host); + if(resolvedHostAllocatorID != selectedHostAllocId) + { + return false; + } + + int* buffer = axom::allocate(ARRAY_SIZE, resolvedHostAllocatorID); + if(buffer == nullptr) + { + return false; + } + + const bool allocatorMatches = axom::getAllocatorIDFromPointer(buffer) == expectedAllocId; + axom::deallocate(buffer); + return allocatorMatches; +} + +TEST(core_memory_management, host_space_allocation_uses_umpire_host_default) +{ + EXPECT_EXIT(([]() { + if(!hostAllocationUsesExpectedAllocator( + axom::getUmpireResourceAllocatorID(umpire::resource::Host), + axom::getUmpireResourceAllocatorID(umpire::resource::Host))) + { + std::exit(1); + } + + std::exit(0); + })(), + ::testing::ExitedWithCode(0), + ""); +} + +TEST(core_memory_management, host_space_allocation_uses_malloc_host_default) +{ + EXPECT_EXIT( + ([]() { + if(!hostAllocationUsesExpectedAllocator(axom::MALLOC_ALLOCATOR_ID, axom::MALLOC_ALLOCATOR_ID)) + { + std::exit(1); + } + + std::exit(0); + })(), + ::testing::ExitedWithCode(0), + ""); +} + +TEST(core_memory_management, host_space_allocation_ignores_global_default_allocator) +{ + EXPECT_EXIT( + ([]() { + const int hostAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Host); + axom::setDefaultHostAllocator(axom::MemorySpace::Malloc); + axom::setDefaultAllocator(axom::MemorySpace::Host); + + if(axom::getDefaultAllocatorID() != hostAllocatorID) + { + std::exit(1); + } + + if(!hostAllocationUsesExpectedAllocator(axom::MALLOC_ALLOCATOR_ID, axom::MALLOC_ALLOCATOR_ID)) + { + std::exit(1); + } + + std::exit(0); + })(), + ::testing::ExitedWithCode(0), + ""); +} + +TEST(core_memory_management, changing_default_host_allocator_after_host_allocation_fails) +{ + EXPECT_DEATH_IF_SUPPORTED( + []() { + const int hostAllocatorID = + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); + axom::setDefaultHostAllocator(hostAllocatorID); + int* buffer = + axom::allocate(ARRAY_SIZE, axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Host)); + AXOM_UNUSED_VAR(buffer); + axom::setDefaultHostAllocator(axom::MALLOC_ALLOCATOR_ID); + }(), + "Default host allocator cannot be changed"); +} TEST(core_memory_management, set_get_default_memory_space) { - const int HostAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Host); - EXPECT_EQ(HostAllocatorID, axom::getDefaultAllocatorID()); + ScopedDefaultAllocatorState scopedState; + const int HostAllocatorID = axom::getDefaultHostAllocatorID(); + const int platformHostAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Host); + EXPECT_EQ(HostAllocatorID, axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Host)); #if defined(AXOM_USE_GPU) - #ifdef UMPIRE_ENABLE_PINNED - const int PinnedAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Pinned); - - axom::setDefaultAllocator(PinnedAllocatorID); - EXPECT_EQ(PinnedAllocatorID, axom::getDefaultAllocatorID()); + #if defined(UMPIRE_ENABLE_PINNED) + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Pinned)) + { + const int PinnedAllocatorID = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Pinned); + axom::setDefaultAllocator(axom::MemorySpace::Pinned); + EXPECT_EQ(PinnedAllocatorID, axom::getDefaultAllocatorID()); + } #endif - #ifdef UMPIRE_ENABLE_DEVICE - const int DeviceAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Device); - axom::setDefaultAllocator(DeviceAllocatorID); - EXPECT_EQ(DeviceAllocatorID, axom::getDefaultAllocatorID()); + #if defined(UMPIRE_ENABLE_DEVICE) + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Device)) + { + const int DeviceAllocatorID = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Device); + axom::setDefaultAllocator(axom::MemorySpace::Device); + EXPECT_EQ(DeviceAllocatorID, axom::getDefaultAllocatorID()); + } #endif - #ifdef UMPIRE_ENABLE_CONST - const int ConstantAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Constant); - axom::setDefaultAllocator(ConstantAllocatorID); - EXPECT_EQ(ConstantAllocatorID, axom::getDefaultAllocatorID()); + #if defined(UMPIRE_ENABLE_CONST) + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Constant)) + { + const int ConstantAllocatorID = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Constant); + axom::setDefaultAllocator(axom::MemorySpace::Constant); + EXPECT_EQ(ConstantAllocatorID, axom::getDefaultAllocatorID()); + } #endif - #ifdef UMPIRE_ENABLE_UM - const int UnifiedAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Unified); - axom::setDefaultAllocator(UnifiedAllocatorID); - EXPECT_EQ(UnifiedAllocatorID, axom::getDefaultAllocatorID()); + #if defined(UMPIRE_ENABLE_UM) + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Unified)) + { + const int UnifiedAllocatorID = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Unified); + axom::setDefaultAllocator(axom::MemorySpace::Unified); + EXPECT_EQ(UnifiedAllocatorID, axom::getDefaultAllocatorID()); + } #endif #endif // AXOM_USE_GPU - axom::setDefaultAllocator(HostAllocatorID); - EXPECT_EQ(HostAllocatorID, axom::getDefaultAllocatorID()); + axom::setDefaultAllocator(axom::MemorySpace::Host); + EXPECT_EQ(platformHostAllocatorID, axom::getDefaultAllocatorID()); } #endif /* AXOM_USE_UMPIRE */ //------------------------------------------------------------------------------ TEST(core_memory_management, alloc_free) { -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) constexpr bool HOST_ACCESSIBLE = true; - const int HostAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Host); + const int HostAllocatorID = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Host); check_alloc_and_free(HostAllocatorID, HOST_ACCESSIBLE); #if defined(AXOM_USE_GPU) constexpr bool NOT_HOST_ACCESSIBLE = false; - #ifdef UMPIRE_ENABLE_PINNED - const int PinnedAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Pinned); - check_alloc_and_free(PinnedAllocatorID, HOST_ACCESSIBLE); + #if defined(UMPIRE_ENABLE_PINNED) + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Pinned)) + { + const int PinnedAllocatorID = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Pinned); + check_alloc_and_free(PinnedAllocatorID, HOST_ACCESSIBLE); + } #endif - #ifdef UMPIRE_ENABLE_DEVICE - const int DeviceAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Device); - check_alloc_and_free(DeviceAllocatorID, NOT_HOST_ACCESSIBLE); + #if defined(UMPIRE_ENABLE_DEVICE) + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Device)) + { + const int DeviceAllocatorID = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Device); + check_alloc_and_free(DeviceAllocatorID, NOT_HOST_ACCESSIBLE); + } #endif - #ifdef UMPIRE_ENABLE_CONST - const int ConstantAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Constant); - check_alloc_and_free(ConstantAllocatorID, NOT_HOST_ACCESSIBLE); + #if defined(UMPIRE_ENABLE_CONST) + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Constant)) + { + const int ConstantAllocatorID = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Constant); + check_alloc_and_free(ConstantAllocatorID, NOT_HOST_ACCESSIBLE); + } #endif - #ifdef UMPIRE_ENABLE_UM - const int UnifiedAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Unified); - check_alloc_and_free(UnifiedAllocatorID, HOST_ACCESSIBLE); + #if defined(UMPIRE_ENABLE_UM) + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Unified)) + { + const int UnifiedAllocatorID = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Unified); + check_alloc_and_free(UnifiedAllocatorID, HOST_ACCESSIBLE); + } #endif #endif // AXOM_USE_GPU @@ -341,36 +586,43 @@ TEST(core_memory_management, alloc_free) //------------------------------------------------------------------------------ TEST(core_memory_management, alloc_realloc_free) { -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) constexpr bool HOST_ACCESSIBLE = true; - const int HostAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Host); + const int HostAllocatorID = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Host); check_alloc_realloc_free(HostAllocatorID, HOST_ACCESSIBLE); #if defined(AXOM_USE_GPU) constexpr bool NOT_HOST_ACCESSIBLE = false; - #ifdef UMPIRE_ENABLE_PINNED - const int PinnedAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Pinned); - check_alloc_realloc_free(PinnedAllocatorID, HOST_ACCESSIBLE); + #if defined(UMPIRE_ENABLE_PINNED) + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Pinned)) + { + const int PinnedAllocatorID = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Pinned); + check_alloc_realloc_free(PinnedAllocatorID, HOST_ACCESSIBLE); + } #endif - #ifdef UMPIRE_ENABLE_DEVICE - const int DeviceAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Device); - check_alloc_realloc_free(DeviceAllocatorID, NOT_HOST_ACCESSIBLE); + #if defined(UMPIRE_ENABLE_DEVICE) + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Device)) + { + const int DeviceAllocatorID = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Device); + check_alloc_realloc_free(DeviceAllocatorID, NOT_HOST_ACCESSIBLE); + } #endif // Umpire doesn't allow reallocation of Constant memory. // check_alloc_realloc_free( axom::getAllocator( umpire::resource::Constant ), // false ); - #ifdef UMPIRE_ENABLE_UM - - const int UnifiedAllocatorID = axom::getUmpireResourceAllocatorID(umpire::resource::Unified); - check_alloc_realloc_free(UnifiedAllocatorID, HOST_ACCESSIBLE); - + #if defined(UMPIRE_ENABLE_UM) + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Unified)) + { + const int UnifiedAllocatorID = axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Unified); + check_alloc_realloc_free(UnifiedAllocatorID, HOST_ACCESSIBLE); + } #endif #endif /* AXOM_USE_GPU */ @@ -404,31 +656,37 @@ TEST_P(CopyTest, Copy) } } -const std::string copy_locations[] = {"NEW", - "MALLOC", - "STATIC" +std::vector copyLocations() +{ + std::vector locations {"NEW", "MALLOC", "STATIC"}; #if defined(AXOM_USE_UMPIRE) - , - "HOST" + locations.push_back("HOST"); #if defined(UMPIRE_ENABLE_DEVICE) - , - "DEVICE" + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Device)) + { + locations.push_back("DEVICE"); + } #endif #if defined(UMPIRE_ENABLE_UM) - , - "UM" + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Unified)) + { + locations.push_back("UM"); + } #endif #if defined(UMPIRE_ENABLE_PINNED) - , - "PINNED" + if(axom::utilities::runtimeMemorySpaceAvailable(axom::MemorySpace::Pinned)) + { + locations.push_back("PINNED"); + } #endif #endif -}; + return locations; +} INSTANTIATE_TEST_SUITE_P(core_memory_management, CopyTest, - ::testing::Combine(::testing::ValuesIn(copy_locations), - ::testing::ValuesIn(copy_locations))); + ::testing::Combine(::testing::ValuesIn(copyLocations()), + ::testing::ValuesIn(copyLocations()))); //------------------------------------------------------------------------------ TEST(core_memory_management, basic_alloc_realloc_dealloc) @@ -470,7 +728,7 @@ TEST(core_memory_management, basic_alloc_realloc_dealloc) } //------------------------------------------------------------------------------ -#ifdef AXOM_USE_UMPIRE +#if defined(AXOM_USE_UMPIRE) TEST(core_memory_management, allocator_id_from_pointer) { constexpr std::size_t N = 5; @@ -490,92 +748,68 @@ TEST(core_memory_management, allocator_id_from_pointer) EXPECT_EQ(id, axom::MALLOC_ALLOCATOR_ID); std::free(buf); } -#endif -//------------------------------------------------------------------------------ -TEST(core_memory_management, interspace_reallocation) +TEST(core_memory_management, foreign_malloc_to_umpire_reallocate_fails) { - // Allocator ids to test. - std::vector allocIds(1, axom::MALLOC_ALLOCATOR_ID); -#ifdef AXOM_USE_UMPIRE - allocIds.push_back(axom::detail::getAllocatorID()); - #ifdef AXOM_USE_GPU - allocIds.push_back(axom::detail::getAllocatorID()); - allocIds.push_back(axom::detail::getAllocatorID()); - // Does it make sense to check Pinned and Constant memory spaces? - #endif -#endif - - // We'll allocate N items, reallocate to K items, reallocate back to N. - constexpr std::size_t N = 5; - constexpr std::size_t K = 8; - constexpr std::size_t maxNK = std::max(N, K); - constexpr std::size_t minNK = std::min(N, K); - - // origOnHost and tempOnHost are for initialization and results-checking on host. - int* origOnHost = axom::allocate(maxNK, axom::MALLOC_ALLOCATOR_ID); - for(std::size_t i = 0; i < maxNK; ++i) - { - origOnHost[i] = static_cast(100 + i); - } - int* tempOnHost = axom::allocate(maxNK, axom::MALLOC_ALLOCATOR_ID); - - // Count differences between origOnHost and tempOnHost. - auto countDiffs = [=]() { - std::size_t diffCount = 0; - for(std::size_t j = 0; j < minNK; ++j) - { - diffCount += tempOnHost[j] != origOnHost[j]; - } - return diffCount; - }; - - std::size_t diffCount = 0; - for(auto srcAllocId : allocIds) - { - for(auto dstAllocId : allocIds) - { - std::cout << "Testing allocator ids " << srcAllocId << " and " << dstAllocId << std::endl; - // For each combination of srcAllocId and dstAllocId, - // allocate src, reallocate to dst, reallocate back to src. - - int* src = axom::allocate(N, srcAllocId); - axom::copy(src, origOnHost, N * sizeof(int)); - - int* dst = axom::reallocate(src, K, dstAllocId); - axom::copy(tempOnHost, dst, N * sizeof(int)); - diffCount = countDiffs(); - EXPECT_EQ(diffCount, 0); - - src = axom::reallocate(dst, N, srcAllocId); - axom::copy(tempOnHost, src, N * sizeof(int)); - diffCount = countDiffs(); - EXPECT_EQ(diffCount, 0); + EXPECT_DEATH_IF_SUPPORTED( + []() { + constexpr std::size_t localN = 5; + const int localUmpireHostAllocId = + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); + int* foreignBuffer = static_cast(std::malloc(localN * sizeof(int))); + axom::reallocate(foreignBuffer, localN + 1, localUmpireHostAllocId); + }(), + "Cannot reallocate across allocator backends"); +} - axom::deallocate(src); - } - } +TEST(core_memory_management, axom_malloc_to_umpire_reallocate_fails) +{ + EXPECT_DEATH_IF_SUPPORTED( + []() { + constexpr std::size_t localN = 5; + const int localUmpireHostAllocId = + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); + int* buffer = axom::allocate(localN, axom::MALLOC_ALLOCATOR_ID); + axom::reallocate(buffer, localN + 1, localUmpireHostAllocId); + }(), + "Cannot reallocate across allocator backends"); +} - axom::deallocate(origOnHost); - axom::deallocate(tempOnHost); +TEST(core_memory_management, umpire_to_axom_malloc_reallocate_fails) +{ + EXPECT_DEATH_IF_SUPPORTED( + []() { + constexpr std::size_t localN = 5; + const int localUmpireHostAllocId = + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); + int* buffer = axom::allocate(localN, localUmpireHostAllocId); + axom::reallocate(buffer, localN + 1, axom::MALLOC_ALLOCATOR_ID); + }(), + "Cannot reallocate across allocator backends"); } +#endif //------------------------------------------------------------------------------ TEST(core_memory_management, test_fill) { // Allocator ids to test. std::vector allocIds(1, axom::MALLOC_ALLOCATOR_ID); -#ifdef AXOM_USE_UMPIRE - allocIds.push_back(axom::detail::getAllocatorID()); +#if defined(AXOM_USE_UMPIRE) + appendAllocatorIDIfAvailable(allocIds, axom::MemorySpace::Host); #ifdef AXOM_USE_GPU - allocIds.push_back(axom::detail::getAllocatorID()); - allocIds.push_back(axom::detail::getAllocatorID()); + #if defined(UMPIRE_ENABLE_DEVICE) + appendAllocatorIDIfAvailable(allocIds, axom::MemorySpace::Device); + #endif + #if defined(UMPIRE_ENABLE_UM) + appendAllocatorIDIfAvailable(allocIds, axom::MemorySpace::Unified); + #endif // Does it make sense to check Pinned and Constant memory spaces? #endif #endif constexpr std::size_t N = 500; int* hostData = axom::allocate(N, axom::MALLOC_ALLOCATOR_ID); + const axom::HostAllocator hostAllocator {axom::MALLOC_ALLOCATOR_ID}; int iteration = 0; for(auto allocId : allocIds) { @@ -595,6 +829,18 @@ TEST(core_memory_management, test_fill) EXPECT_EQ(hostData[i], fillValue); } + const int explicitFillValue = 23456 + iteration; + axom::fill(buffer, N, explicitFillValue, hostAllocator); + + // Copy back to host + axom::copy(hostData, buffer, N * sizeof(int)); + + // Make sure elements have the right fill value. + for(std::size_t i = 0; i < N; i++) + { + EXPECT_EQ(hostData[i], explicitFillValue); + } + axom::deallocate(buffer); iteration++; } diff --git a/src/axom/core/utilities/MemoryTesting.hpp b/src/axom/core/utilities/MemoryTesting.hpp new file mode 100644 index 0000000000..05e9f99add --- /dev/null +++ b/src/axom/core/utilities/MemoryTesting.hpp @@ -0,0 +1,98 @@ +// Copyright (c) Lawrence Livermore National Security, LLC and other +// Axom Project Contributors. See top-level LICENSE and COPYRIGHT +// files for dates and other details. +// +// SPDX-License-Identifier: (BSD-3-Clause) + +/*! + * + * \file MemoryTesting.hpp + * + * \brief Header file containing utility functions for memory spaces in Axom tests + * + */ + +#pragma once + +#include "axom/config.hpp" +#include "axom/core/execution/execution_space.hpp" +#include "axom/core/memory_management.hpp" + +#include + +namespace axom +{ +namespace utilities +{ + +template +int globalDefaultAllocatorForExecSpace() +{ +#if defined(AXOM_USE_UMPIRE) + return axom::execution_space::onDevice() + ? axom::execution_space::allocatorID() + : axom::getUmpireResourceAllocatorID(umpire::resource::Host); +#else + return axom::execution_space::allocatorID(); +#endif +} + +inline bool runtimeMemorySpaceAvailable(axom::MemorySpace space) +{ + if(!axom::isMemorySpaceAvailable(space)) + { + return false; + } + +#if defined(AXOM_USE_UMPIRE) + try + { + switch(space) + { + case axom::MemorySpace::Host: + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); + break; + case axom::MemorySpace::Device: + #if defined(UMPIRE_ENABLE_DEVICE) + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Device); + break; + #else + return false; + #endif + case axom::MemorySpace::Unified: + #if defined(UMPIRE_ENABLE_UM) + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Unified); + break; + #else + return false; + #endif + case axom::MemorySpace::Pinned: + #if defined(UMPIRE_ENABLE_PINNED) + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Pinned); + break; + #else + return false; + #endif + case axom::MemorySpace::Constant: + #if defined(UMPIRE_ENABLE_CONST) + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Constant); + break; + #else + return false; + #endif + case axom::MemorySpace::Malloc: + case axom::MemorySpace::Dynamic: + break; + } + } + catch(const std::exception&) + { + return false; + } +#endif + + return true; +} + +} // namespace utilities +} // namespace axom diff --git a/src/axom/mint/docs/sphinx/sections/execution_model.rst b/src/axom/mint/docs/sphinx/sections/execution_model.rst index 67774c0c90..8cc6575230 100644 --- a/src/axom/mint/docs/sphinx/sections/execution_model.rst +++ b/src/axom/mint/docs/sphinx/sections/execution_model.rst @@ -76,6 +76,13 @@ See the :ref:`sections/tutorial` for code snippets that illustrate how to use the :ref:`NodeTraversalFunctions`, :ref:`CellTraversalFunctions` and :ref:`FaceTraversalFunctions` of the :ref:`sections/execution_model`. +Some traversal signatures stage coordinates, connectivity, offsets, or +adjacency data for execution on a device. The traversal APIs provide overloads +that accept ``axom::HostAllocator`` for that host-resident staging. Prefer +those overloads in new code when host allocator ownership is available. +Traversal overloads without a host allocator remain compatibility paths and use +Axom's current default host allocator. + .. _executionPolicy: Execution Policy diff --git a/src/axom/mint/execution/interface.hpp b/src/axom/mint/execution/interface.hpp index 5641c091d8..e8c852229f 100644 --- a/src/axom/mint/execution/interface.hpp +++ b/src/axom/mint/execution/interface.hpp @@ -9,6 +9,7 @@ #include "axom/config.hpp" // compile-time definitions #include "axom/core/Macros.hpp" // for AXOM_STATIC_ASSERT #include "axom/core/execution/execution_space.hpp" // for execution_space traits +#include "axom/core/memory_management.hpp" #include "axom/mint/execution/xargs.hpp" // for xargs #include "axom/mint/execution/internal/for_all_cells.hpp" // for_all_cells() @@ -91,6 +92,11 @@ namespace mint * * \pre m != nullptr * + * \note Overloads that accept `HostAllocator` use it for host staging needed + * by coordinate-rich execution signatures. Overloads without + * `HostAllocator` are compatibility paths that use Axom's current + * default host allocator. + * * \tparam ExecPolicy the execution policy, e.g., serial or parallel * \tparam ArgType object indicating the arguments to the kernel * \tparam KernelType @@ -146,6 +152,26 @@ inline void for_all_nodes(const MeshType* m, KernelType&& kernel) internal::for_all_nodes_impl(ArgType(), *m, std::forward(kernel)); } +template +inline void for_all_nodes(const MeshType* m, HostAllocator hostAllocator, KernelType&& kernel) +{ + // compile-time sanity checks + AXOM_STATIC_ASSERT(execution_space::valid()); + AXOM_STATIC_ASSERT(xargs_traits::valid()); + + constexpr bool valid_mesh_type = std::is_base_of::value; + AXOM_STATIC_ASSERT(valid_mesh_type); + + // run-time sanity checks + SLIC_ASSERT(m != nullptr); + + // dispatch + internal::for_all_nodes(ArgType(), + static_cast(*m), + std::forward(kernel), + hostAllocator); +} + template inline void for_all_nodes(const Mesh* m, KernelType&& kernel) { @@ -160,6 +186,20 @@ inline void for_all_nodes(const Mesh* m, KernelType&& kernel) internal::for_all_nodes(ArgType(), *m, std::forward(kernel)); } +template +inline void for_all_nodes(const Mesh* m, HostAllocator hostAllocator, KernelType&& kernel) +{ + // compile-time sanity checks + AXOM_STATIC_ASSERT(execution_space::valid()); + AXOM_STATIC_ASSERT(xargs_traits::valid()); + + // run-time sanity checks + SLIC_ASSERT(m != nullptr); + + //dispatch + internal::for_all_nodes(ArgType(), *m, std::forward(kernel), hostAllocator); +} + /// @} /// @} @@ -175,6 +215,11 @@ inline void for_all_nodes(const Mesh* m, KernelType&& kernel) * * \pre m != nullptr * + * \note Overloads that accept `HostAllocator` use it for host staging needed + * by connectivity, coordinate, offset, or adjacency execution + * signatures. Overloads without `HostAllocator` are compatibility paths + * that use Axom's current default host allocator. + * * \tparam ExecPolicy the execution policy, e.g., serial or parallel * \tparam ArgType object indicating the arguments to the kernel * @@ -236,6 +281,26 @@ inline void for_all_cells(const MeshType* m, KernelType&& kernel) internal::for_all_cells_impl(ArgType(), *m, std::forward(kernel)); } +template +inline void for_all_cells(const MeshType* m, HostAllocator hostAllocator, KernelType&& kernel) +{ + // compile-time sanity checks + AXOM_STATIC_ASSERT(execution_space::valid()); + AXOM_STATIC_ASSERT(xargs_traits::valid()); + + constexpr bool valid_mesh_type = std::is_base_of::value; + AXOM_STATIC_ASSERT(valid_mesh_type); + + // run-time sanity checks + SLIC_ASSERT(m != nullptr); + + // dispatch + internal::for_all_cells(ArgType(), + static_cast(*m), + std::forward(kernel), + hostAllocator); +} + template inline void for_all_cells(const Mesh* m, KernelType&& kernel) { @@ -250,6 +315,20 @@ inline void for_all_cells(const Mesh* m, KernelType&& kernel) internal::for_all_cells(ArgType(), *m, std::forward(kernel)); } +template +inline void for_all_cells(const Mesh* m, HostAllocator hostAllocator, KernelType&& kernel) +{ + // compile-time sanity checks + AXOM_STATIC_ASSERT(execution_space::valid()); + AXOM_STATIC_ASSERT(xargs_traits::valid()); + + // run-time sanity checks + SLIC_ASSERT(m != nullptr); + + //dispatch + internal::for_all_cells(ArgType(), *m, std::forward(kernel), hostAllocator); +} + /// @} /// @} @@ -265,6 +344,11 @@ inline void for_all_cells(const Mesh* m, KernelType&& kernel) * * \pre m != nullptr * + * \note Overloads that accept `HostAllocator` use it for host staging needed + * by connectivity, coordinate, offset, or adjacency execution + * signatures. Overloads without `HostAllocator` are compatibility paths + * that use Axom's current default host allocator. + * * \tparam ExecPolicy the execution policy, e.g., serial or parallel * \tparam ArgType object indicating the arguments to the kernel * @@ -321,6 +405,26 @@ inline void for_all_faces(const MeshType* m, KernelType&& kernel) internal::for_all_faces_impl(ArgType(), *m, std::forward(kernel)); } +template +inline void for_all_faces(const MeshType* m, HostAllocator hostAllocator, KernelType&& kernel) +{ + // compile-time sanity checks + AXOM_STATIC_ASSERT(execution_space::valid()); + AXOM_STATIC_ASSERT(xargs_traits::valid()); + + constexpr bool valid_mesh_type = std::is_base_of::value; + AXOM_STATIC_ASSERT(valid_mesh_type); + + // run-time sanity checks + SLIC_ASSERT(m != nullptr); + + // dispatch + internal::for_all_faces(ArgType(), + static_cast(*m), + std::forward(kernel), + hostAllocator); +} + template inline void for_all_faces(const Mesh* m, KernelType&& kernel) { @@ -335,6 +439,20 @@ inline void for_all_faces(const Mesh* m, KernelType&& kernel) internal::for_all_faces(ArgType(), *m, std::forward(kernel)); } +template +inline void for_all_faces(const Mesh* m, HostAllocator hostAllocator, KernelType&& kernel) +{ + // compile-time sanity checks + AXOM_STATIC_ASSERT(execution_space::valid()); + AXOM_STATIC_ASSERT(xargs_traits::valid()); + + // run-time sanity checks + SLIC_ASSERT(m != nullptr); + + //dispatch + internal::for_all_faces(ArgType(), *m, std::forward(kernel), hostAllocator); +} + /// @} /// @} diff --git a/src/axom/mint/execution/internal/for_all_cells.hpp b/src/axom/mint/execution/internal/for_all_cells.hpp index 869d3f118c..59d9f6b0b7 100644 --- a/src/axom/mint/execution/internal/for_all_cells.hpp +++ b/src/axom/mint/execution/internal/for_all_cells.hpp @@ -10,6 +10,7 @@ #include "axom/config.hpp" // compile time definitions #include "axom/core/execution/execution_space.hpp" // for execution_space traits #include "axom/core/execution/for_all.hpp" // for axom::for_all +#include "axom/core/memory_management.hpp" // mint includes #include "axom/mint/execution/xargs.hpp" // for xargs @@ -48,6 +49,15 @@ inline void for_all_cells(xargs::index, const Mesh& m, KernelType&& kernel) for_all_cells_impl(xargs::index(), m, std::forward(kernel)); } +template +inline void for_all_cells(xargs::index, + const Mesh& m, + KernelType&& kernel, + HostAllocator AXOM_UNUSED_PARAM(hostAllocator)) +{ + for_all_cells(xargs::index(), m, std::forward(kernel)); +} + //------------------------------------------------------------------------------ template inline void for_all_cells_impl(xargs::ij, const StructuredMesh& m, KernelType&& kernel) @@ -80,6 +90,15 @@ inline void for_all_cells(xargs::ij, const Mesh& m, KernelType&& kernel) for_all_cells_impl(xargs::ij(), sm, std::forward(kernel)); } +template +inline void for_all_cells(xargs::ij, + const Mesh& m, + KernelType&& kernel, + HostAllocator AXOM_UNUSED_PARAM(hostAllocator)) +{ + for_all_cells(xargs::ij(), m, std::forward(kernel)); +} + //------------------------------------------------------------------------------ template inline void for_all_cells_impl(xargs::ijk, const StructuredMesh& m, KernelType&& kernel) @@ -113,6 +132,15 @@ inline void for_all_cells(xargs::ijk, const Mesh& m, KernelType&& kernel) for_all_cells_impl(xargs::ijk(), sm, std::forward(kernel)); } +template +inline void for_all_cells(xargs::ijk, + const Mesh& m, + KernelType&& kernel, + HostAllocator AXOM_UNUSED_PARAM(hostAllocator)) +{ + for_all_cells(xargs::ijk(), m, std::forward(kernel)); +} + //------------------------------------------------------------------------------ template inline void for_all_cells_impl(xargs::nodeids, const StructuredMesh& m, KernelType&& kernel) @@ -177,7 +205,8 @@ inline void for_all_cells_impl(xargs::nodeids, const StructuredMesh& m, KernelTy template inline void for_all_cells_impl(xargs::nodeids, const UnstructuredMesh& m, - KernelType&& kernel) + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { constexpr bool on_device = axom::execution_space::onDevice(); const int device_allocator = axom::execution_space::allocatorID(); @@ -189,12 +218,13 @@ inline void for_all_cells_impl(xargs::nodeids, // Move cell connectivity and cell offsets onto device axom::Array cell_connectivity_d = on_device - ? axom::Array(cell_connectivity_h, device_allocator) + ? axom::Array(cell_connectivity_h, device_allocator, hostAllocator) : axom::Array(); auto cell_connectivity_view = on_device ? cell_connectivity_d.view() : cell_connectivity_h; - axom::Array cell_offsets_d = - on_device ? axom::Array(cell_offsets_h, device_allocator) : axom::Array(); + axom::Array cell_offsets_d = on_device + ? axom::Array(cell_offsets_h, device_allocator, hostAllocator) + : axom::Array(); auto cell_offsets_view = on_device ? cell_offsets_d.view() : cell_offsets_h; for_all_cells_impl( @@ -210,7 +240,8 @@ inline void for_all_cells_impl(xargs::nodeids, template inline void for_all_cells_impl(xargs::nodeids, const UnstructuredMesh& m, - KernelType&& kernel) + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { constexpr bool on_device = axom::execution_space::onDevice(); const int device_allocator = axom::execution_space::allocatorID(); @@ -220,7 +251,7 @@ inline void for_all_cells_impl(xargs::nodeids, // Move cell connectivity onto device axom::Array cell_connectivity_d = on_device - ? axom::Array(cell_connectivity_h, device_allocator) + ? axom::Array(cell_connectivity_h, device_allocator, hostAllocator) : axom::Array(); auto cell_connectivity_view = on_device ? cell_connectivity_d.view() : cell_connectivity_h; @@ -236,7 +267,10 @@ inline void for_all_cells_impl(xargs::nodeids, //------------------------------------------------------------------------------ template -inline void for_all_cells(xargs::nodeids, const Mesh& m, KernelType&& kernel) +inline void for_all_cells(xargs::nodeids, + const Mesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { if(m.isStructured()) { @@ -246,12 +280,18 @@ inline void for_all_cells(xargs::nodeids, const Mesh& m, KernelType&& kernel) else if(m.hasMixedCellTypes()) { const UnstructuredMesh& um = static_cast&>(m); - for_all_cells_impl(xargs::nodeids(), um, std::forward(kernel)); + for_all_cells_impl(xargs::nodeids(), + um, + std::forward(kernel), + hostAllocator); } else { const UnstructuredMesh& um = static_cast&>(m); - for_all_cells_impl(xargs::nodeids(), um, std::forward(kernel)); + for_all_cells_impl(xargs::nodeids(), + um, + std::forward(kernel), + hostAllocator); } } @@ -317,7 +357,8 @@ inline void for_all_cells_impl(xargs::faceids, const StructuredMesh& m, KernelTy template inline void for_all_cells_impl(xargs::faceids, const UnstructuredMesh& m, - KernelType&& kernel) + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { constexpr bool on_device = axom::execution_space::onDevice(); const int device_allocator = axom::execution_space::allocatorID(); @@ -330,7 +371,7 @@ inline void for_all_cells_impl(xargs::faceids, // Move cells_to_faces values onto device axom::Array cells_to_faces_d = on_device - ? axom::Array(cells_to_faces_h, device_allocator) + ? axom::Array(cells_to_faces_h, device_allocator, hostAllocator) : axom::Array(); auto cells_to_faces_v = on_device ? cells_to_faces_d.view() : cells_to_faces_h; @@ -346,7 +387,8 @@ inline void for_all_cells_impl(xargs::faceids, template inline void for_all_cells_impl(xargs::faceids, const UnstructuredMesh& m, - KernelType&& kernel) + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { constexpr bool on_device = axom::execution_space::onDevice(); const int device_allocator = axom::execution_space::allocatorID(); @@ -359,12 +401,13 @@ inline void for_all_cells_impl(xargs::faceids, // Move cells_to_faces and offsets values onto device axom::Array cells_to_faces_d = on_device - ? axom::Array(cells_to_faces_h, device_allocator) + ? axom::Array(cells_to_faces_h, device_allocator, hostAllocator) : axom::Array(); auto cells_to_faces_v = on_device ? cells_to_faces_d.view() : cells_to_faces_h; - axom::Array offsets_d = - on_device ? axom::Array(offsets_h, device_allocator) : axom::Array(); + axom::Array offsets_d = on_device + ? axom::Array(offsets_h, device_allocator, hostAllocator) + : axom::Array(); auto offsets_v = on_device ? offsets_d.view() : offsets_h; for_all_cells_impl( @@ -378,7 +421,10 @@ inline void for_all_cells_impl(xargs::faceids, //------------------------------------------------------------------------------ template -inline void for_all_cells(xargs::faceids, const Mesh& m, KernelType&& kernel) +inline void for_all_cells(xargs::faceids, + const Mesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ERROR_IF(m.getDimension() == 1, "For all cells with face IDs only supported for 2D and 3D meshes"); @@ -391,12 +437,18 @@ inline void for_all_cells(xargs::faceids, const Mesh& m, KernelType&& kernel) else if(m.hasMixedCellTypes()) { const UnstructuredMesh& um = static_cast&>(m); - for_all_cells_impl(xargs::faceids(), um, std::forward(kernel)); + for_all_cells_impl(xargs::faceids(), + um, + std::forward(kernel), + hostAllocator); } else { const UnstructuredMesh& um = static_cast&>(m); - for_all_cells_impl(xargs::faceids(), um, std::forward(kernel)); + for_all_cells_impl(xargs::faceids(), + um, + std::forward(kernel), + hostAllocator); } } @@ -492,7 +544,10 @@ inline void for_all_cells_impl(xargs::coords, const UniformMesh& m, KernelType&& //------------------------------------------------------------------------------ template -inline void for_all_cells_impl(xargs::coords, const RectilinearMesh& m, KernelType&& kernel) +inline void for_all_cells_impl(xargs::coords, + const RectilinearMesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { constexpr bool NO_COPY = true; @@ -508,8 +563,9 @@ inline void for_all_cells_impl(xargs::coords, const RectilinearMesh& m, KernelTy m.getNodeResolution(X_COORDINATE)); // Move x values onto device - axom::Array x_vals_d = - on_device ? axom::Array(x_vals_h, device_allocator) : axom::Array(); + axom::Array x_vals_d = on_device + ? axom::Array(x_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto x_vals_view = on_device ? x_vals_d.view() : x_vals_h; if(dimension == 1) @@ -532,8 +588,9 @@ inline void for_all_cells_impl(xargs::coords, const RectilinearMesh& m, KernelTy m.getNodeResolution(Y_COORDINATE)); // Move y values onto device - axom::Array y_vals_d = - on_device ? axom::Array(y_vals_h, device_allocator) : axom::Array(); + axom::Array y_vals_d = on_device + ? axom::Array(y_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto y_vals_view = on_device ? y_vals_d.view() : y_vals_h; for_all_cells_impl( @@ -567,10 +624,12 @@ inline void for_all_cells_impl(xargs::coords, const RectilinearMesh& m, KernelTy m.getNodeResolution(Z_COORDINATE)); // Move yz values onto device - axom::Array y_vals_d = - on_device ? axom::Array(y_vals_h, device_allocator) : axom::Array(); - axom::Array z_vals_d = - on_device ? axom::Array(z_vals_h, device_allocator) : axom::Array(); + axom::Array y_vals_d = on_device + ? axom::Array(y_vals_h, device_allocator, hostAllocator) + : axom::Array(); + axom::Array z_vals_d = on_device + ? axom::Array(z_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto y_vals_view = on_device ? y_vals_d.view() : y_vals_h; auto z_vals_view = on_device ? z_vals_d.view() : z_vals_h; @@ -609,43 +668,54 @@ struct for_all_cell_nodes_functor template inline void operator()(ExecPolicy AXOM_UNUSED_PARAM(policy), const MeshType& m, + HostAllocator hostAllocator, KernelType&& kernel) const { constexpr bool valid_mesh_type = std::is_base_of::value; AXOM_STATIC_ASSERT(valid_mesh_type); + AXOM_UNUSED_VAR(hostAllocator); for_all_cells_impl(xargs::nodeids(), m, std::forward(kernel)); } }; //------------------------------------------------------------------------------ template -inline void for_all_cells_impl(xargs::coords, const CurvilinearMesh& m, KernelType&& kernel) +inline void for_all_cells_impl(xargs::coords, + const CurvilinearMesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { const int dimension = m.getDimension(); if(dimension == 1) { for_all_coords(for_all_cell_nodes_functor(), m, + hostAllocator, std::forward(kernel)); } else if(dimension == 2) { for_all_coords(for_all_cell_nodes_functor(), m, + hostAllocator, std::forward(kernel)); } else { for_all_coords(for_all_cell_nodes_functor(), m, + hostAllocator, std::forward(kernel)); } } //------------------------------------------------------------------------------ template -inline void for_all_cells_impl(xargs::coords, const UnstructuredMesh& m, KernelType&& kernel) +inline void for_all_cells_impl(xargs::coords, + const UnstructuredMesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { constexpr bool NO_COPY = true; @@ -659,8 +729,9 @@ inline void for_all_cells_impl(xargs::coords, const UnstructuredMesh& m, K auto x_vals_h = axom::ArrayView(m.getCoordinateArray(X_COORDINATE), coordinate_size); // Move x values onto device - axom::Array x_vals_d = - on_device ? axom::Array(x_vals_h, device_allocator) : axom::Array(); + axom::Array x_vals_d = on_device + ? axom::Array(x_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto x_vals_view = on_device ? x_vals_d.view() : x_vals_h; if(dimension == 1) @@ -673,7 +744,8 @@ inline void for_all_cells_impl(xargs::coords, const UnstructuredMesh& m, K numerics::Matrix coordsMatrix(dimension, 2, coords, NO_COPY); kernel(cellID, coordsMatrix, nodeIDs); - }); + }, + hostAllocator); } else if(dimension == 2) { @@ -682,8 +754,9 @@ inline void for_all_cells_impl(xargs::coords, const UnstructuredMesh& m, K axom::ArrayView(m.getCoordinateArray(Y_COORDINATE), coordinate_size); // Move y values onto device - axom::Array y_vals_d = - on_device ? axom::Array(y_vals_h, device_allocator) : axom::Array(); + axom::Array y_vals_d = on_device + ? axom::Array(y_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto y_vals_view = on_device ? y_vals_d.view() : y_vals_h; for_all_cells_impl( @@ -700,7 +773,8 @@ inline void for_all_cells_impl(xargs::coords, const UnstructuredMesh& m, K numerics::Matrix coordsMatrix(dimension, numNodes, coords, NO_COPY); kernel(cellID, coordsMatrix, nodeIDs); - }); + }, + hostAllocator); } else { @@ -713,12 +787,14 @@ inline void for_all_cells_impl(xargs::coords, const UnstructuredMesh& m, K axom::ArrayView(m.getCoordinateArray(Z_COORDINATE), coordinate_size); // Move yz values onto device - axom::Array y_vals_d = - on_device ? axom::Array(y_vals_h, device_allocator) : axom::Array(); + axom::Array y_vals_d = on_device + ? axom::Array(y_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto y_vals_view = on_device ? y_vals_d.view() : y_vals_h; - axom::Array z_vals_d = - on_device ? axom::Array(z_vals_h, device_allocator) : axom::Array(); + axom::Array z_vals_d = on_device + ? axom::Array(z_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto z_vals_view = on_device ? z_vals_d.view() : z_vals_h; for_all_cells_impl( @@ -736,13 +812,17 @@ inline void for_all_cells_impl(xargs::coords, const UnstructuredMesh& m, K numerics::Matrix coordsMatrix(dimension, numNodes, coords, NO_COPY); kernel(cellID, coordsMatrix, nodeIDs); - }); + }, + hostAllocator); } } //------------------------------------------------------------------------------ template -inline void for_all_cells(xargs::coords, const Mesh& m, KernelType&& kernel) +inline void for_all_cells(xargs::coords, + const Mesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { if(m.getMeshType() == STRUCTURED_UNIFORM_MESH) { @@ -752,12 +832,12 @@ inline void for_all_cells(xargs::coords, const Mesh& m, KernelType&& kernel) else if(m.getMeshType() == STRUCTURED_RECTILINEAR_MESH) { const RectilinearMesh& rm = static_cast(m); - for_all_cells_impl(xargs::coords(), rm, std::forward(kernel)); + for_all_cells_impl(xargs::coords(), rm, std::forward(kernel), hostAllocator); } else if(m.getMeshType() == STRUCTURED_CURVILINEAR_MESH) { const CurvilinearMesh& cm = static_cast(m); - for_all_cells_impl(xargs::coords(), cm, std::forward(kernel)); + for_all_cells_impl(xargs::coords(), cm, std::forward(kernel), hostAllocator); } else if(m.getMeshType() == UNSTRUCTURED_MESH) { @@ -765,14 +845,20 @@ inline void for_all_cells(xargs::coords, const Mesh& m, KernelType&& kernel) { const UnstructuredMesh& um = static_cast&>(m); - for_all_cells_impl(xargs::coords(), um, std::forward(kernel)); + for_all_cells_impl(xargs::coords(), + um, + std::forward(kernel), + hostAllocator); } else { const UnstructuredMesh& um = static_cast&>(m); - for_all_cells_impl(xargs::coords(), um, std::forward(kernel)); + for_all_cells_impl(xargs::coords(), + um, + std::forward(kernel), + hostAllocator); } } else diff --git a/src/axom/mint/execution/internal/for_all_faces.hpp b/src/axom/mint/execution/internal/for_all_faces.hpp index c24963e73e..a19f38b423 100644 --- a/src/axom/mint/execution/internal/for_all_faces.hpp +++ b/src/axom/mint/execution/internal/for_all_faces.hpp @@ -10,6 +10,7 @@ #include "axom/config.hpp" // compile time definitions #include "axom/core/execution/execution_space.hpp" // for execution_space traits #include "axom/core/execution/for_all.hpp" // for axom::for_all +#include "axom/core/memory_management.hpp" // mint includes #include "axom/mint/execution/xargs.hpp" // for xargs @@ -165,6 +166,15 @@ inline void for_all_faces(xargs::index, const Mesh& m, KernelType&& kernel) return for_all_faces_impl(xargs::index(), m, std::forward(kernel)); } +template +inline void for_all_faces(xargs::index, + const Mesh& m, + KernelType&& kernel, + HostAllocator AXOM_UNUSED_PARAM(hostAllocator)) +{ + return for_all_faces(xargs::index(), m, std::forward(kernel)); +} + //------------------------------------------------------------------------------ template inline void for_all_faces_impl(xargs::nodeids, const StructuredMesh& m, KernelType&& kernel) @@ -261,7 +271,8 @@ inline void for_all_faces_impl(xargs::nodeids, const StructuredMesh& m, KernelTy template inline void for_all_faces_impl(xargs::nodeids, const UnstructuredMesh& m, - KernelType&& kernel) + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ERROR_IF(m.getNumberOfFaces() <= 0, "No faces in the mesh, perhaps you meant to call " @@ -275,7 +286,7 @@ inline void for_all_faces_impl(xargs::nodeids, // Move faces to nodes onto device axom::Array faces_to_nodes_d = on_device - ? axom::Array(faces_to_nodes_h, device_allocator) + ? axom::Array(faces_to_nodes_h, device_allocator, hostAllocator) : axom::Array(); auto faces_to_nodes_view = on_device ? faces_to_nodes_d.view() : faces_to_nodes_h; @@ -294,7 +305,8 @@ inline void for_all_faces_impl(xargs::nodeids, template inline void for_all_faces_impl(xargs::nodeids, const UnstructuredMesh& m, - KernelType&& kernel) + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ERROR_IF(m.getNumberOfFaces() <= 0, "No faces in the mesh, perhaps you meant to call " @@ -310,10 +322,11 @@ inline void for_all_faces_impl(xargs::nodeids, // Move faces to nodes and offsets onto device axom::Array faces_to_nodes_d = on_device - ? axom::Array(faces_to_nodes_h, device_allocator) + ? axom::Array(faces_to_nodes_h, device_allocator, hostAllocator) + : axom::Array(); + axom::Array offsets_d = on_device + ? axom::Array(offsets_h, device_allocator, hostAllocator) : axom::Array(); - axom::Array offsets_d = - on_device ? axom::Array(offsets_h, device_allocator) : axom::Array(); auto faces_to_nodes_view = on_device ? faces_to_nodes_d.view() : faces_to_nodes_h; auto offsets_view = on_device ? offsets_d.view() : offsets_h; @@ -329,7 +342,10 @@ inline void for_all_faces_impl(xargs::nodeids, //------------------------------------------------------------------------------ template -inline void for_all_faces(xargs::nodeids, const Mesh& m, KernelType&& kernel) +inline void for_all_faces(xargs::nodeids, + const Mesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ASSERT(m.getDimension() > 1 && m.getDimension() <= 3); @@ -341,12 +357,18 @@ inline void for_all_faces(xargs::nodeids, const Mesh& m, KernelType&& kernel) else if(m.hasMixedCellTypes()) { const UnstructuredMesh& um = static_cast&>(m); - for_all_faces_impl(xargs::nodeids(), um, std::forward(kernel)); + for_all_faces_impl(xargs::nodeids(), + um, + std::forward(kernel), + hostAllocator); } else { const UnstructuredMesh& um = static_cast&>(m); - for_all_faces_impl(xargs::nodeids(), um, std::forward(kernel)); + for_all_faces_impl(xargs::nodeids(), + um, + std::forward(kernel), + hostAllocator); } } @@ -467,7 +489,10 @@ inline void for_all_faces_impl(xargs::cellids, const StructuredMesh& m, KernelTy //------------------------------------------------------------------------------ template -inline void for_all_faces_impl(xargs::cellids, const UnstructuredMesh& m, KernelType&& kernel) +inline void for_all_faces_impl(xargs::cellids, + const UnstructuredMesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ERROR_IF(m.getNumberOfFaces() <= 0, "No faces in the mesh, perhaps you meant to call " @@ -481,7 +506,7 @@ inline void for_all_faces_impl(xargs::cellids, const UnstructuredMesh& m, // Move faces to cells onto device axom::Array faces_to_cells_d = on_device - ? axom::Array(faces_to_cells_h, device_allocator) + ? axom::Array(faces_to_cells_h, device_allocator, hostAllocator) : axom::Array(); auto faces_to_cells_view = on_device ? faces_to_cells_d.view() : faces_to_cells_h; @@ -497,7 +522,10 @@ inline void for_all_faces_impl(xargs::cellids, const UnstructuredMesh& m, //------------------------------------------------------------------------------ template -inline void for_all_faces(xargs::cellids, const Mesh& m, KernelType&& kernel) +inline void for_all_faces(xargs::cellids, + const Mesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ASSERT(m.getDimension() > 1 && m.getDimension() <= 3); @@ -509,12 +537,18 @@ inline void for_all_faces(xargs::cellids, const Mesh& m, KernelType&& kernel) else if(m.hasMixedCellTypes()) { const UnstructuredMesh& um = static_cast&>(m); - for_all_faces_impl(xargs::cellids(), um, std::forward(kernel)); + for_all_faces_impl(xargs::cellids(), + um, + std::forward(kernel), + hostAllocator); } else { const UnstructuredMesh& um = static_cast&>(m); - for_all_faces_impl(xargs::cellids(), um, std::forward(kernel)); + for_all_faces_impl(xargs::cellids(), + um, + std::forward(kernel), + hostAllocator); } } @@ -647,7 +681,10 @@ inline void for_all_faces_impl(xargs::coords, const UniformMesh& m, KernelType&& //------------------------------------------------------------------------------ template -inline void for_all_faces_impl(xargs::coords, const RectilinearMesh& m, KernelType&& kernel) +inline void for_all_faces_impl(xargs::coords, + const RectilinearMesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { constexpr bool NO_COPY = true; @@ -666,10 +703,12 @@ inline void for_all_faces_impl(xargs::coords, const RectilinearMesh& m, KernelTy m.getNodeResolution(Y_COORDINATE)); // Move xy values onto device - axom::Array x_vals_d = - on_device ? axom::Array(x_vals_h, device_allocator) : axom::Array(); - axom::Array y_vals_d = - on_device ? axom::Array(y_vals_h, device_allocator) : axom::Array(); + axom::Array x_vals_d = on_device + ? axom::Array(x_vals_h, device_allocator, hostAllocator) + : axom::Array(); + axom::Array y_vals_d = on_device + ? axom::Array(y_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto x_vals_view = on_device ? x_vals_d.view() : x_vals_h; auto y_vals_view = on_device ? y_vals_d.view() : y_vals_h; @@ -708,8 +747,9 @@ inline void for_all_faces_impl(xargs::coords, const RectilinearMesh& m, KernelTy m.getNodeResolution(Z_COORDINATE)); // Move z values onto device - axom::Array z_vals_d = - on_device ? axom::Array(z_vals_h, device_allocator) : axom::Array(); + axom::Array z_vals_d = on_device + ? axom::Array(z_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto z_vals_view = on_device ? z_vals_d.view() : z_vals_h; @@ -793,18 +833,23 @@ struct for_all_face_nodes_functor template inline void operator()(ExecPolicy AXOM_UNUSED_PARAM(policy), const MeshType& m, + HostAllocator hostAllocator, KernelType&& kernel) const { constexpr bool valid_mesh_type = std::is_base_of::value; AXOM_STATIC_ASSERT(valid_mesh_type); + AXOM_UNUSED_VAR(hostAllocator); for_all_faces_impl(xargs::nodeids(), m, std::forward(kernel)); } }; //------------------------------------------------------------------------------ template -inline void for_all_faces_impl(xargs::coords, const CurvilinearMesh& m, KernelType&& kernel) +inline void for_all_faces_impl(xargs::coords, + const CurvilinearMesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ASSERT(m.getDimension() > 1 && m.getDimension() <= 3); @@ -813,19 +858,24 @@ inline void for_all_faces_impl(xargs::coords, const CurvilinearMesh& m, KernelTy { for_all_coords(for_all_face_nodes_functor(), m, + hostAllocator, std::forward(kernel)); } else { for_all_coords(for_all_face_nodes_functor(), m, + hostAllocator, std::forward(kernel)); } } //------------------------------------------------------------------------------ template -inline void for_all_faces_impl(xargs::coords, const UnstructuredMesh& m, KernelType&& kernel) +inline void for_all_faces_impl(xargs::coords, + const UnstructuredMesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { constexpr bool NO_COPY = true; @@ -842,10 +892,12 @@ inline void for_all_faces_impl(xargs::coords, const UnstructuredMesh& m, K auto y_vals_h = axom::ArrayView(m.getCoordinateArray(Y_COORDINATE), coordinate_size); // Move xy values onto device - axom::Array x_vals_d = - on_device ? axom::Array(x_vals_h, device_allocator) : axom::Array(); - axom::Array y_vals_d = - on_device ? axom::Array(y_vals_h, device_allocator) : axom::Array(); + axom::Array x_vals_d = on_device + ? axom::Array(x_vals_h, device_allocator, hostAllocator) + : axom::Array(); + axom::Array y_vals_d = on_device + ? axom::Array(y_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto x_vals_view = on_device ? x_vals_d.view() : x_vals_h; auto y_vals_view = on_device ? y_vals_d.view() : y_vals_h; @@ -866,7 +918,8 @@ inline void for_all_faces_impl(xargs::coords, const UnstructuredMesh& m, K numerics::Matrix coordsMatrix(dimension, 2, coords, NO_COPY); kernel(faceID, coordsMatrix, nodeIDs); - }); + }, + hostAllocator); } else { @@ -874,8 +927,9 @@ inline void for_all_faces_impl(xargs::coords, const UnstructuredMesh& m, K axom::ArrayView(m.getCoordinateArray(Z_COORDINATE), coordinate_size); // Move z values onto device - axom::Array z_vals_d = - on_device ? axom::Array(z_vals_h, device_allocator) : axom::Array(); + axom::Array z_vals_d = on_device + ? axom::Array(z_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto z_vals_view = on_device ? z_vals_d.view() : z_vals_h; @@ -894,13 +948,17 @@ inline void for_all_faces_impl(xargs::coords, const UnstructuredMesh& m, K numerics::Matrix coordsMatrix(dimension, numNodes, coords, NO_COPY); kernel(faceID, coordsMatrix, nodeIDs); - }); + }, + hostAllocator); } } //------------------------------------------------------------------------------ template -inline void for_all_faces(xargs::coords, const Mesh& m, KernelType&& kernel) +inline void for_all_faces(xargs::coords, + const Mesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ERROR_IF(m.getDimension() <= 1 || m.getDimension() > 3, "Invalid dimension"); @@ -912,12 +970,12 @@ inline void for_all_faces(xargs::coords, const Mesh& m, KernelType&& kernel) else if(m.getMeshType() == STRUCTURED_RECTILINEAR_MESH) { const RectilinearMesh& rm = static_cast(m); - for_all_faces_impl(xargs::coords(), rm, std::forward(kernel)); + for_all_faces_impl(xargs::coords(), rm, std::forward(kernel), hostAllocator); } else if(m.getMeshType() == STRUCTURED_CURVILINEAR_MESH) { const CurvilinearMesh& cm = static_cast(m); - for_all_faces_impl(xargs::coords(), cm, std::forward(kernel)); + for_all_faces_impl(xargs::coords(), cm, std::forward(kernel), hostAllocator); } else if(m.getMeshType() == UNSTRUCTURED_MESH) { @@ -925,14 +983,20 @@ inline void for_all_faces(xargs::coords, const Mesh& m, KernelType&& kernel) { const UnstructuredMesh& um = static_cast&>(m); - for_all_faces_impl(xargs::coords(), um, std::forward(kernel)); + for_all_faces_impl(xargs::coords(), + um, + std::forward(kernel), + hostAllocator); } else { const UnstructuredMesh& um = static_cast&>(m); - for_all_faces_impl(xargs::coords(), um, std::forward(kernel)); + for_all_faces_impl(xargs::coords(), + um, + std::forward(kernel), + hostAllocator); } } else diff --git a/src/axom/mint/execution/internal/for_all_nodes.hpp b/src/axom/mint/execution/internal/for_all_nodes.hpp index bae3807dca..e57a95c41b 100644 --- a/src/axom/mint/execution/internal/for_all_nodes.hpp +++ b/src/axom/mint/execution/internal/for_all_nodes.hpp @@ -10,6 +10,7 @@ #include "axom/config.hpp" // compile time definitions #include "axom/core/execution/execution_space.hpp" // for execution_space traits #include "axom/core/execution/for_all.hpp" // for axom::for_all +#include "axom/core/memory_management.hpp" // mint includes #include "axom/mint/execution/xargs.hpp" // for xargs @@ -42,6 +43,15 @@ inline void for_all_nodes(xargs::index, const Mesh& m, KernelType&& kernel) for_all_nodes_impl(xargs::index(), m, std::forward(kernel)); } +template +inline void for_all_nodes(xargs::index, + const Mesh& m, + KernelType&& kernel, + HostAllocator AXOM_UNUSED_PARAM(hostAllocator)) +{ + for_all_nodes(xargs::index(), m, std::forward(kernel)); +} + //------------------------------------------------------------------------------ template inline void for_all_nodes_impl(xargs::ij, const StructuredMesh& m, KernelType&& kernel) @@ -73,6 +83,15 @@ inline void for_all_nodes(xargs::ij, const Mesh& m, KernelType&& kernel) for_all_nodes_impl(xargs::ij(), sm, std::forward(kernel)); } +template +inline void for_all_nodes(xargs::ij, + const Mesh& m, + KernelType&& kernel, + HostAllocator AXOM_UNUSED_PARAM(hostAllocator)) +{ + for_all_nodes(xargs::ij(), m, std::forward(kernel)); +} + //------------------------------------------------------------------------------ template inline void for_all_nodes_impl(xargs::ijk, const StructuredMesh& m, KernelType&& kernel) @@ -107,6 +126,15 @@ inline void for_all_nodes(xargs::ijk, const Mesh& m, KernelType&& kernel) for_all_nodes_impl(xargs::ijk(), sm, std::forward(kernel)); } +template +inline void for_all_nodes(xargs::ijk, + const Mesh& m, + KernelType&& kernel, + HostAllocator AXOM_UNUSED_PARAM(hostAllocator)) +{ + for_all_nodes(xargs::ijk(), m, std::forward(kernel)); +} + //------------------------------------------------------------------------------ template inline void for_all_nodes_impl(xargs::x, const UniformMesh& m, KernelType&& kernel) @@ -127,7 +155,10 @@ inline void for_all_nodes_impl(xargs::x, const UniformMesh& m, KernelType&& kern //------------------------------------------------------------------------------ template -inline void for_all_nodes_impl(xargs::x, const Mesh& m, KernelType&& kernel) +inline void for_all_nodes_impl(xargs::x, + const Mesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ERROR_IF(m.getDimension() != 1, "xargs::x is only valid for 1D meshes"); SLIC_ERROR_IF(m.getMeshType() == STRUCTURED_UNIFORM_MESH, "Not valid for UniformMesh."); @@ -141,8 +172,9 @@ inline void for_all_nodes_impl(xargs::x, const Mesh& m, KernelType&& kernel) SLIC_ASSERT(x_vals_h.data() != nullptr); // Move x values onto device - axom::Array x_vals_d = - on_device ? axom::Array(x_vals_h, device_allocator) : axom::Array(); + axom::Array x_vals_d = on_device + ? axom::Array(x_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto x_vals_view = on_device ? x_vals_d.view() : x_vals_h; for_all_nodes_impl( @@ -153,7 +185,10 @@ inline void for_all_nodes_impl(xargs::x, const Mesh& m, KernelType&& kernel) //------------------------------------------------------------------------------ template -inline void for_all_nodes(xargs::x, const Mesh& m, KernelType&& kernel) +inline void for_all_nodes(xargs::x, + const Mesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ERROR_IF(m.getDimension() != 1, "xargs::x is only valid for 1D meshes"); @@ -165,13 +200,16 @@ inline void for_all_nodes(xargs::x, const Mesh& m, KernelType&& kernel) } else { - for_all_nodes_impl(xargs::x(), m, std::forward(kernel)); + for_all_nodes_impl(xargs::x(), m, std::forward(kernel), hostAllocator); } } //------------------------------------------------------------------------------ template -inline void for_all_nodes_impl(xargs::xy, const UniformMesh& m, KernelType&& kernel) +inline void for_all_nodes_impl(xargs::xy, + const UniformMesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ERROR_IF(m.getDimension() != 2, "xargs::xy is only valid for 2D meshes"); constexpr bool on_device = axom::execution_space::onDevice(); @@ -182,12 +220,14 @@ inline void for_all_nodes_impl(xargs::xy, const UniformMesh& m, KernelType&& ker auto spacing_h = axom::ArrayView(m.getSpacing().begin(), 3); // Move origin and spacing values onto device - axom::Array origin_d = - on_device ? axom::Array(origin_h, device_allocator) : axom::Array(); + axom::Array origin_d = on_device + ? axom::Array(origin_h, device_allocator, hostAllocator) + : axom::Array(); auto origin_view = on_device ? origin_d.view() : origin_h; - axom::Array spacing_d = - on_device ? axom::Array(spacing_h, device_allocator) : axom::Array(); + axom::Array spacing_d = on_device + ? axom::Array(spacing_h, device_allocator, hostAllocator) + : axom::Array(); auto spacing_view = on_device ? spacing_d.view() : spacing_h; for_all_nodes_impl( @@ -202,7 +242,10 @@ inline void for_all_nodes_impl(xargs::xy, const UniformMesh& m, KernelType&& ker //------------------------------------------------------------------------------ template -inline void for_all_nodes_impl(xargs::xy, const RectilinearMesh& m, KernelType&& kernel) +inline void for_all_nodes_impl(xargs::xy, + const RectilinearMesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ERROR_IF(m.getDimension() != 2, "xargs::xy is only valid for 2D meshes"); @@ -219,12 +262,14 @@ inline void for_all_nodes_impl(xargs::xy, const RectilinearMesh& m, KernelType&& SLIC_ASSERT(y_vals_h.data() != nullptr); // Move xy values onto device - axom::Array x_vals_d = - on_device ? axom::Array(x_vals_h, device_allocator) : axom::Array(); + axom::Array x_vals_d = on_device + ? axom::Array(x_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto x_vals_view = on_device ? x_vals_d.view() : x_vals_h; - axom::Array y_vals_d = - on_device ? axom::Array(y_vals_h, device_allocator) : axom::Array(); + axom::Array y_vals_d = on_device + ? axom::Array(y_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto y_vals_view = on_device ? y_vals_d.view() : y_vals_h; for_all_nodes_impl( @@ -237,7 +282,10 @@ inline void for_all_nodes_impl(xargs::xy, const RectilinearMesh& m, KernelType&& //------------------------------------------------------------------------------ template -inline void for_all_nodes_impl(xargs::xy, const Mesh& m, KernelType&& kernel) +inline void for_all_nodes_impl(xargs::xy, + const Mesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ERROR_IF(m.getDimension() != 2, "xargs::xy is only valid for 2D meshes"); SLIC_ERROR_IF(m.getMeshType() == STRUCTURED_UNIFORM_MESH, "Not valid for UniformMesh."); @@ -255,12 +303,14 @@ inline void for_all_nodes_impl(xargs::xy, const Mesh& m, KernelType&& kernel) SLIC_ASSERT(y_vals_h.data() != nullptr); // Move xy values onto device - axom::Array x_vals_d = - on_device ? axom::Array(x_vals_h, device_allocator) : axom::Array(); + axom::Array x_vals_d = on_device + ? axom::Array(x_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto x_vals_view = on_device ? x_vals_d.view() : x_vals_h; - axom::Array y_vals_d = - on_device ? axom::Array(y_vals_h, device_allocator) : axom::Array(); + axom::Array y_vals_d = on_device + ? axom::Array(y_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto y_vals_view = on_device ? y_vals_d.view() : y_vals_h; for_all_nodes_impl( @@ -271,7 +321,10 @@ inline void for_all_nodes_impl(xargs::xy, const Mesh& m, KernelType&& kernel) //------------------------------------------------------------------------------ template -inline void for_all_nodes(xargs::xy, const Mesh& m, KernelType&& kernel) +inline void for_all_nodes(xargs::xy, + const Mesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ERROR_IF(m.getDimension() != 2, "xargs::xy is only valid for 3D meshes"); @@ -279,22 +332,25 @@ inline void for_all_nodes(xargs::xy, const Mesh& m, KernelType&& kernel) if(mesh_type == STRUCTURED_RECTILINEAR_MESH) { const RectilinearMesh& rm = static_cast(m); - for_all_nodes_impl(xargs::xy(), rm, std::forward(kernel)); + for_all_nodes_impl(xargs::xy(), rm, std::forward(kernel), hostAllocator); } else if(mesh_type == STRUCTURED_UNIFORM_MESH) { const UniformMesh& um = static_cast(m); - for_all_nodes_impl(xargs::xy(), um, std::forward(kernel)); + for_all_nodes_impl(xargs::xy(), um, std::forward(kernel), hostAllocator); } else { - for_all_nodes_impl(xargs::xy(), m, std::forward(kernel)); + for_all_nodes_impl(xargs::xy(), m, std::forward(kernel), hostAllocator); } } //------------------------------------------------------------------------------ template -inline void for_all_nodes_impl(xargs::xyz, const UniformMesh& m, KernelType&& kernel) +inline void for_all_nodes_impl(xargs::xyz, + const UniformMesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ERROR_IF(m.getDimension() != 3, "xargs::xyz is only valid for 3D meshes"); constexpr bool on_device = axom::execution_space::onDevice(); @@ -305,12 +361,14 @@ inline void for_all_nodes_impl(xargs::xyz, const UniformMesh& m, KernelType&& ke auto spacing_h = axom::ArrayView(m.getSpacing().begin(), 3); // Move origin and spacing values onto device - axom::Array origin_d = - on_device ? axom::Array(origin_h, device_allocator) : axom::Array(); + axom::Array origin_d = on_device + ? axom::Array(origin_h, device_allocator, hostAllocator) + : axom::Array(); auto origin_view = on_device ? origin_d.view() : origin_h; - axom::Array spacing_d = - on_device ? axom::Array(spacing_h, device_allocator) : axom::Array(); + axom::Array spacing_d = on_device + ? axom::Array(spacing_h, device_allocator, hostAllocator) + : axom::Array(); auto spacing_view = on_device ? spacing_d.view() : spacing_h; for_all_nodes_impl( @@ -326,7 +384,10 @@ inline void for_all_nodes_impl(xargs::xyz, const UniformMesh& m, KernelType&& ke //------------------------------------------------------------------------------ template -inline void for_all_nodes_impl(xargs::xyz, const RectilinearMesh& m, KernelType&& kernel) +inline void for_all_nodes_impl(xargs::xyz, + const RectilinearMesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ERROR_IF(m.getDimension() != 3, "xargs::xyz is only valid for 3D meshes"); constexpr bool on_device = axom::execution_space::onDevice(); @@ -341,16 +402,19 @@ inline void for_all_nodes_impl(xargs::xyz, const RectilinearMesh& m, KernelType& m.getNodeResolution(Z_COORDINATE)); // Move xyz values onto device - axom::Array x_vals_d = - on_device ? axom::Array(x_vals_h, device_allocator) : axom::Array(); + axom::Array x_vals_d = on_device + ? axom::Array(x_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto x_vals_view = on_device ? x_vals_d.view() : x_vals_h; - axom::Array y_vals_d = - on_device ? axom::Array(y_vals_h, device_allocator) : axom::Array(); + axom::Array y_vals_d = on_device + ? axom::Array(y_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto y_vals_view = on_device ? y_vals_d.view() : y_vals_h; - axom::Array z_vals_d = - on_device ? axom::Array(z_vals_h, device_allocator) : axom::Array(); + axom::Array z_vals_d = on_device + ? axom::Array(z_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto z_vals_view = on_device ? z_vals_d.view() : z_vals_h; for_all_nodes_impl( @@ -363,7 +427,10 @@ inline void for_all_nodes_impl(xargs::xyz, const RectilinearMesh& m, KernelType& //------------------------------------------------------------------------------ template -inline void for_all_nodes_impl(xargs::xyz, const Mesh& m, KernelType&& kernel) +inline void for_all_nodes_impl(xargs::xyz, + const Mesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ERROR_IF(m.getDimension() != 3, "xargs::xyz is only valid for 3D meshes"); SLIC_ERROR_IF(m.getMeshType() == STRUCTURED_UNIFORM_MESH, "Not valid for UniformMesh."); @@ -383,16 +450,19 @@ inline void for_all_nodes_impl(xargs::xyz, const Mesh& m, KernelType&& kernel) SLIC_ASSERT(z_vals_h.data() != nullptr); // Move xyz values onto device - axom::Array x_vals_d = - on_device ? axom::Array(x_vals_h, device_allocator) : axom::Array(); + axom::Array x_vals_d = on_device + ? axom::Array(x_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto x_vals_view = on_device ? x_vals_d.view() : x_vals_h; - axom::Array y_vals_d = - on_device ? axom::Array(y_vals_h, device_allocator) : axom::Array(); + axom::Array y_vals_d = on_device + ? axom::Array(y_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto y_vals_view = on_device ? y_vals_d.view() : y_vals_h; - axom::Array z_vals_d = - on_device ? axom::Array(z_vals_h, device_allocator) : axom::Array(); + axom::Array z_vals_d = on_device + ? axom::Array(z_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto z_vals_view = on_device ? z_vals_d.view() : z_vals_h; #if !defined(AXOM_USE_RAJA) && defined(AXOM_USE_CUDA) // If we have CUDA but not RAJA then we are doing serial execution and cannot @@ -413,7 +483,10 @@ inline void for_all_nodes_impl(xargs::xyz, const Mesh& m, KernelType&& kernel) //------------------------------------------------------------------------------ template -inline void for_all_nodes(xargs::xyz, const mint::Mesh& m, KernelType&& kernel) +inline void for_all_nodes(xargs::xyz, + const mint::Mesh& m, + KernelType&& kernel, + HostAllocator hostAllocator = HostAllocator {}) { SLIC_ERROR_IF(m.getDimension() != 3, "xargs::xyz is only valid for 3D meshes"); @@ -421,16 +494,16 @@ inline void for_all_nodes(xargs::xyz, const mint::Mesh& m, KernelType&& kernel) if(mesh_type == STRUCTURED_RECTILINEAR_MESH) { const RectilinearMesh& rm = static_cast(m); - for_all_nodes_impl(xargs::xyz(), rm, std::forward(kernel)); + for_all_nodes_impl(xargs::xyz(), rm, std::forward(kernel), hostAllocator); } else if(mesh_type == STRUCTURED_UNIFORM_MESH) { const UniformMesh& um = static_cast(m); - for_all_nodes_impl(xargs::xyz(), um, std::forward(kernel)); + for_all_nodes_impl(xargs::xyz(), um, std::forward(kernel), hostAllocator); } else { - for_all_nodes_impl(xargs::xyz(), m, std::forward(kernel)); + for_all_nodes_impl(xargs::xyz(), m, std::forward(kernel), hostAllocator); } } diff --git a/src/axom/mint/execution/internal/helpers.hpp b/src/axom/mint/execution/internal/helpers.hpp index af65d10b04..cfff2fa078 100644 --- a/src/axom/mint/execution/internal/helpers.hpp +++ b/src/axom/mint/execution/internal/helpers.hpp @@ -11,6 +11,7 @@ #include "axom/mint/mesh/Mesh.hpp" // for Mesh #include "axom/core/Macros.hpp" +#include "axom/core/memory_management.hpp" #include "axom/core/StackArray.hpp" // for axom::StackArray #include "axom/core/numerics/Matrix.hpp" // for Matrix @@ -41,7 +42,10 @@ namespace internal */ template -inline void for_all_coords(const FOR_ALL_FUNCTOR& for_all_nodes, const MeshType& m, KernelType&& kernel) +inline void for_all_coords(const FOR_ALL_FUNCTOR& for_all_nodes, + const MeshType& m, + HostAllocator hostAllocator, + KernelType&& kernel) { SLIC_ERROR_IF(m.getMeshType() == STRUCTURED_UNIFORM_MESH, "Not valid for UniformMesh."); SLIC_ERROR_IF(m.getMeshType() == STRUCTURED_RECTILINEAR_MESH, "Not valid for RectilinearMesh."); @@ -70,20 +74,23 @@ inline void for_all_coords(const FOR_ALL_FUNCTOR& for_all_nodes, const MeshType& : axom::ArrayView(); // Move xyz values onto device - axom::Array x_vals_d = axom::Array(x_vals_h, device_allocator); + axom::Array x_vals_d = axom::Array(x_vals_h, device_allocator, hostAllocator); auto x_vals_view = x_vals_d.view(); - axom::Array y_vals_d = - (NDIM > 1) ? axom::Array(y_vals_h, device_allocator) : axom::Array(); + axom::Array y_vals_d = (NDIM > 1) + ? axom::Array(y_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto y_vals_view = (NDIM > 1) ? y_vals_d.view() : y_vals_h; - axom::Array z_vals_d = - (NDIM > 2) ? axom::Array(z_vals_h, device_allocator) : axom::Array(); + axom::Array z_vals_d = (NDIM > 2) + ? axom::Array(z_vals_h, device_allocator, hostAllocator) + : axom::Array(); auto z_vals_view = (NDIM > 2) ? z_vals_d.view() : z_vals_h; for_all_nodes( ExecPolicy(), m, + hostAllocator, AXOM_LAMBDA(IndexType objectID, const IndexType* nodeIDs, IndexType numNodes) { AXOM_UNUSED_VAR(numNodes); assert(numNodes == NNODES); @@ -109,6 +116,15 @@ inline void for_all_coords(const FOR_ALL_FUNCTOR& for_all_nodes, const MeshType& }); } +template +inline void for_all_coords(const FOR_ALL_FUNCTOR& for_all_nodes, const MeshType& m, KernelType&& kernel) +{ + for_all_coords(for_all_nodes, + m, + HostAllocator {}, + std::forward(kernel)); +} + } /* namespace internal */ } /* namespace mint */ } /* namespace axom */ diff --git a/src/axom/primal/tests/primal_clip.cpp b/src/axom/primal/tests/primal_clip.cpp index 7ca6f4adad..f479f026e3 100644 --- a/src/axom/primal/tests/primal_clip.cpp +++ b/src/axom/primal/tests/primal_clip.cpp @@ -11,6 +11,7 @@ #include "axom/core/execution/for_all.hpp" #include "axom/core/memory_management.hpp" #include "axom/core/numerics/transforms.hpp" +#include "axom/core/utilities/MemoryTesting.hpp" #include "axom/primal/geometry/Point.hpp" #include "axom/primal/geometry/BoundingBox.hpp" @@ -333,7 +334,7 @@ void unit_check_poly_clip() // In addition, vertices 0 and 3 should be marked as clipped. const int current_allocator = axom::getDefaultAllocatorID(); - axom::setDefaultAllocator(axom::execution_space::allocatorID()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); PolyhedronType* out_square = axom::allocate(1); unsigned int* out_clipped = axom::allocate(1); @@ -431,7 +432,7 @@ void check_hex_tet_clip(double EPS) const int current_allocator = axom::getDefaultAllocatorID(); // Set new default to device if available - axom::setDefaultAllocator(axom::execution_space::allocatorID()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); // Allocate memory for shapes TetrahedronType* tet = axom::allocate(1); @@ -510,7 +511,7 @@ void check_oct_tet_clip(double EPS) const int current_allocator = axom::getDefaultAllocatorID(); // Set new default to device if available - axom::setDefaultAllocator(axom::execution_space::allocatorID()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); // Allocate memory for shapes TetrahedronType* tet = axom::allocate(1); @@ -586,7 +587,7 @@ void check_tet_tet_clip(double EPS) const int current_allocator = axom::getDefaultAllocatorID(); // Set new default to device if available - axom::setDefaultAllocator(axom::execution_space::allocatorID()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); // Allocate memory for shapes TetrahedronType* tet1 = axom::allocate(1); diff --git a/src/axom/primal/tests/primal_intersect.cpp b/src/axom/primal/tests/primal_intersect.cpp index 66fd2c2bca..8b6a3970d5 100644 --- a/src/axom/primal/tests/primal_intersect.cpp +++ b/src/axom/primal/tests/primal_intersect.cpp @@ -10,6 +10,7 @@ #include "axom/core/execution/execution_space.hpp" #include "axom/core/memory_management.hpp" +#include "axom/core/utilities/MemoryTesting.hpp" #include "axom/primal/geometry/OrientedBoundingBox.hpp" #include "axom/primal/geometry/BoundingBox.hpp" @@ -30,6 +31,7 @@ namespace primal = axom::primal; namespace { + template primal::Point randomPt(double beg, double end) { @@ -2590,14 +2592,7 @@ void check_plane_bb_intersect() // Save current/default allocator const int current_allocator = axom::getDefaultAllocatorID(); - // Determine new allocator (for CUDA or HIP policy, set to device) - umpire::Allocator allocator = - (axom::execution_space::onDevice() - ? rm.getAllocator(umpire::resource::Device) - : rm.getAllocator(axom::execution_space::allocatorID())); - - // Set new default to device - axom::setDefaultAllocator(allocator.getId()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); // Initialize bounding box and planes on device, // intersection results in unified memory to check results on host. @@ -2671,14 +2666,7 @@ void check_plane_seg_intersect() // Save current/default allocator const int current_allocator = axom::getDefaultAllocatorID(); - // Determine new allocator (for CUDA or HIP policy, set to device) - umpire::Allocator allocator = - (axom::execution_space::onDevice() - ? rm.getAllocator(umpire::resource::Device) - : rm.getAllocator(axom::execution_space::allocatorID())); - - // Set new default to device - axom::setDefaultAllocator(allocator.getId()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); // Initialize planes and segments on device, // intersection results in unified memory to check results on host. @@ -2761,12 +2749,7 @@ void check_segment_segment_intersect_policy() const int current_allocator = axom::getDefaultAllocatorID(); - umpire::Allocator allocator = - (axom::execution_space::onDevice() - ? rm.getAllocator(umpire::resource::Device) - : rm.getAllocator(axom::execution_space::allocatorID())); - - axom::setDefaultAllocator(allocator.getId()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); const int result_allocator = (axom::execution_space::onDevice() ? rm.getAllocator(umpire::resource::Unified).getId() diff --git a/src/axom/primal/tests/primal_zip.cpp b/src/axom/primal/tests/primal_zip.cpp index 7c33b510a4..e4d9052829 100644 --- a/src/axom/primal/tests/primal_zip.cpp +++ b/src/axom/primal/tests/primal_zip.cpp @@ -9,6 +9,8 @@ #include "axom/core/execution/for_all.hpp" #include "axom/core/execution/runtime_policy.hpp" #include "axom/core/memory_management.hpp" +#include "axom/core/utilities/MemoryTesting.hpp" + #include "axom/slic.hpp" #include "axom/primal/geometry/Point.hpp" @@ -38,7 +40,7 @@ void check_zip_points_3d() using ZipType = primal::ZipIndexable; const int current_allocator = axom::getDefaultAllocatorID(); - axom::setDefaultAllocator(axom::execution_space::allocatorID()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); // create arrays of data constexpr int N = 8; @@ -86,7 +88,7 @@ void check_zip_points_2d_from_3d() using ZipType = primal::ZipIndexable; const int current_allocator = axom::getDefaultAllocatorID(); - axom::setDefaultAllocator(axom::execution_space::allocatorID()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); // create arrays of data constexpr int N = 4; @@ -132,7 +134,7 @@ void check_zip_vectors_2d_from_3d() using ZipType = primal::ZipIndexable; const int current_allocator = axom::getDefaultAllocatorID(); - axom::setDefaultAllocator(axom::execution_space::allocatorID()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); // create arrays of data constexpr int N = 4; @@ -179,7 +181,7 @@ void check_zip_bbs_3d() using ZipType = primal::ZipIndexable; const int current_allocator = axom::getDefaultAllocatorID(); - axom::setDefaultAllocator(axom::execution_space::allocatorID()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); // create arrays of data constexpr int N = 8; @@ -246,7 +248,7 @@ void check_zip_bbs_2d_from_3d() using ZipType = primal::ZipIndexable; const int current_allocator = axom::getDefaultAllocatorID(); - axom::setDefaultAllocator(axom::execution_space::allocatorID()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); // create arrays of data constexpr int N = 4; @@ -310,7 +312,7 @@ void check_zip_rays_3d() using ZipType = primal::ZipIndexable; const int current_allocator = axom::getDefaultAllocatorID(); - axom::setDefaultAllocator(axom::execution_space::allocatorID()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); // create arrays of data constexpr int N = 4; @@ -389,7 +391,7 @@ void check_zip_rays_2d_from_3d() using ZipType = primal::ZipIndexable; const int current_allocator = axom::getDefaultAllocatorID(); - axom::setDefaultAllocator(axom::execution_space::allocatorID()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); // create arrays of data constexpr int N = 4; diff --git a/src/axom/quest/DiscreteShape.cpp b/src/axom/quest/DiscreteShape.cpp index 456aa502c8..93d80dbe04 100644 --- a/src/axom/quest/DiscreteShape.cpp +++ b/src/axom/quest/DiscreteShape.cpp @@ -31,8 +31,16 @@ constexpr double DiscreteShape::DEFAULT_VERTEX_WELD_THRESHOLD; DiscreteShape::DiscreteShape(const axom::klee::Shape& shape, axom::sidre::Group* parentGroup, const std::string& prefixPath) + : DiscreteShape(shape, parentGroup, HostAllocator {}, prefixPath) +{ } + +DiscreteShape::DiscreteShape(const axom::klee::Shape& shape, + axom::sidre::Group* parentGroup, + HostAllocator hostAllocator, + const std::string& prefixPath) : m_shape(shape) , m_sidreGroup(nullptr) + , m_hostAllocator(hostAllocator) , m_refinementType(DiscreteShape::RefinementUniformSegments) , m_percentError(utilities::clampVal(0.0, MINIMUM_PERCENT_ERROR, MAXIMUM_PERCENT_ERROR)) { @@ -269,11 +277,7 @@ void DiscreteShape::createRepresentationOfBlueprintTets() // Put the in-memory geometry in m_meshRep. const axom::sidre::Group* inputGroup = geometry.getBlueprintMesh(); -#ifdef AXOM_USE_UMPIRE - int allocID = inputGroup->getDefaultAllocatorID(); -#else - int allocID = axom::execution_space::allocatorID(); -#endif + const int allocID = m_hostAllocator.getID(); std::string modName = inputGroup->getName() + "_modified"; while(m_sidreGroup->hasGroup(modName)) @@ -558,7 +562,8 @@ void DiscreteShape::createRepresentationOfSOR() int(polyline.size()), m_shape.getGeometry().getLevelOfRefinement(), octs, - octCount); + octCount, + m_hostAllocator); AXOM_UNUSED_VAR(good); SLIC_ASSERT(good); diff --git a/src/axom/quest/DiscreteShape.hpp b/src/axom/quest/DiscreteShape.hpp index 8147e09a63..56b7c9e323 100644 --- a/src/axom/quest/DiscreteShape.hpp +++ b/src/axom/quest/DiscreteShape.hpp @@ -10,6 +10,7 @@ #include #include "axom/config.hpp" +#include "axom/core/memory_management.hpp" #include "axom/klee/Shape.hpp" #include "axom/mint/mesh/UnstructuredMesh.hpp" @@ -65,6 +66,15 @@ class DiscreteShape axom::sidre::Group* parentGroup, const std::string& prefixPath = {}); + /*! + @brief Constructor with explicit host allocator for host-resident scratch + and staging allocations. + */ + DiscreteShape(const axom::klee::Shape& shape, + axom::sidre::Group* parentGroup, + HostAllocator hostAllocator, + const std::string& prefixPath = {}); + virtual ~DiscreteShape() { clearInternalData(); } ///@{ @@ -143,6 +153,9 @@ class DiscreteShape //!@brief Sidre store for m_meshRep. axom::sidre::Group* m_sidreGroup {nullptr}; + //!@brief Host allocator for host-resident scratch and staging allocations. + HostAllocator m_hostAllocator; + //!@brief Prefix for disc files with relative path. std::string m_prefixPath; diff --git a/src/axom/quest/Discretize.hpp b/src/axom/quest/Discretize.hpp index bf06bb8f94..d93687a1f8 100644 --- a/src/axom/quest/Discretize.hpp +++ b/src/axom/quest/Discretize.hpp @@ -8,6 +8,7 @@ // Axom includes #include "axom/core/Macros.hpp" +#include "axom/core/memory_management.hpp" // Geometry #include "axom/primal/geometry/Sphere.hpp" @@ -71,7 +72,8 @@ bool discretize(const axom::ArrayView& polyline, int len, int levels, axom::Array& out, - int& octcount); + int& octcount, + HostAllocator hostAllocator = HostAllocator {}); /// @} diff --git a/src/axom/quest/DistributedClosestPoint.cpp b/src/axom/quest/DistributedClosestPoint.cpp index d6d1b84869..d7caa45c29 100644 --- a/src/axom/quest/DistributedClosestPoint.cpp +++ b/src/axom/quest/DistributedClosestPoint.cpp @@ -100,6 +100,16 @@ void DistributedClosestPoint::setAllocatorID(int allocatorID) } } +void DistributedClosestPoint::setHostAllocator(HostAllocator hostAllocator) +{ + m_hostAllocator = hostAllocator; + + if(m_impl) + { + m_impl->setHostAllocator(m_hostAllocator); + } +} + void DistributedClosestPoint::setMpiCommunicator(MPI_Comm mpiComm, bool duplicate) { if(m_mpiCommIsPrivate) @@ -262,6 +272,7 @@ template void DistributedClosestPoint::allocateQueryInstance() { m_impl = std::make_unique>(m_allocatorID, + m_hostAllocator, m_isVerbose); } diff --git a/src/axom/quest/DistributedClosestPoint.hpp b/src/axom/quest/DistributedClosestPoint.hpp index 0151a843c4..b2f77de120 100644 --- a/src/axom/quest/DistributedClosestPoint.hpp +++ b/src/axom/quest/DistributedClosestPoint.hpp @@ -7,6 +7,7 @@ #pragma once #include "axom/config.hpp" +#include "axom/core/memory_management.hpp" #include "axom/core/execution/runtime_policy.hpp" #include "axom/slic.hpp" @@ -86,6 +87,14 @@ class DistributedClosestPoint */ void setAllocatorID(int allocatorID); + /*! + * \brief Sets the host allocator used for host-resident scratch and staging. + * + * If not explicitly set, this defaults to Axom's legacy host allocator + * convenience path. + */ + void setHostAllocator(HostAllocator hostAllocator); + /** * \brief Set the MPI communicator. * @@ -195,6 +204,7 @@ class DistributedClosestPoint MPI_Comm m_mpiComm; bool m_mpiCommIsPrivate; int m_allocatorID; + HostAllocator m_hostAllocator {}; int m_dimension {-1}; bool m_isVerbose {false}; double m_sqDistanceThreshold; diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index ee70481b59..17029d051a 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -342,6 +342,18 @@ class IntersectionShaper : public Shaper const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) : Shaper(runtimePolicy, allocatorId, shapeSet, dc) + , m_hostAllocator {} + { + m_free_mat_name = "free"; + } + + IntersectionShaper(RuntimePolicy runtimePolicy, + int allocatorId, + HostAllocator hostAllocator, + const klee::ShapeSet& shapeSet, + sidre::MFEMSidreDataCollection* dc) + : Shaper(runtimePolicy, allocatorId, hostAllocator, shapeSet, dc) + , m_hostAllocator(hostAllocator) { m_free_mat_name = "free"; } @@ -363,6 +375,18 @@ class IntersectionShaper : public Shaper sidre::Group* bpGrp, const std::string& topo = "") : Shaper(runtimePolicy, allocatorId, shapeSet, bpGrp, topo) + , m_hostAllocator {} + , m_free_mat_name("free") + { } + + IntersectionShaper(RuntimePolicy runtimePolicy, + int allocatorId, + HostAllocator hostAllocator, + const klee::ShapeSet& shapeSet, + sidre::Group* bpGrp, + const std::string& topo = "") + : Shaper(runtimePolicy, allocatorId, hostAllocator, shapeSet, bpGrp, topo) + , m_hostAllocator(hostAllocator) , m_free_mat_name("free") { } @@ -376,6 +400,18 @@ class IntersectionShaper : public Shaper conduit::Node& bpNode, const std::string& topo = "") : Shaper(runtimePolicy, allocatorId, shapeSet, bpNode, topo) + , m_hostAllocator {} + , m_free_mat_name("free") + { } + + IntersectionShaper(RuntimePolicy runtimePolicy, + int allocatorId, + HostAllocator hostAllocator, + const klee::ShapeSet& shapeSet, + conduit::Node& bpNode, + const std::string& topo = "") + : Shaper(runtimePolicy, allocatorId, hostAllocator, shapeSet, bpNode, topo) + , m_hostAllocator(hostAllocator) , m_free_mat_name("free") { } #endif @@ -457,7 +493,7 @@ class IntersectionShaper : public Shaper auto quads_device_view = m_quads.view(); AXOM_ANNOTATE_BEGIN("allocate m_cell_volumes"); - m_cell_volumes = axom::Array(m_cellCount, m_cellCount, m_allocatorId); + m_cell_volumes = axom::Array(m_cellCount, m_cellCount, m_allocatorId, m_hostAllocator); AXOM_ANNOTATE_END("allocate m_cell_volumes"); m_cell_volumes.fill(0.0); @@ -470,7 +506,7 @@ class IntersectionShaper : public Shaper AXOM_ANNOTATE_END("cell_volume"); AXOM_ANNOTATE_BEGIN("populate m_quad_bbs"); - m_quad_bbs = axom::Array(m_cellCount, m_cellCount, m_allocatorId); + m_quad_bbs = axom::Array(m_cellCount, m_cellCount, m_allocatorId, m_hostAllocator); axom::ArrayView quad_bbs_device_view = m_quad_bbs.view(); // Get bounding boxes for quadrilateral elements @@ -490,7 +526,7 @@ class IntersectionShaper : public Shaper AXOM_ANNOTATE_END("populate m_quad_bbs"); AXOM_ANNOTATE_BEGIN("allocate m_overlap_volumes"); - m_overlap_volumes = axom::Array(m_cellCount, m_cellCount, m_allocatorId); + m_overlap_volumes = axom::Array(m_cellCount, m_cellCount, m_allocatorId, m_hostAllocator); AXOM_ANNOTATE_END("allocate m_overlap_volumes"); } @@ -509,14 +545,15 @@ class IntersectionShaper : public Shaper m_tets_from_hexes_device = axom::Array(ArrayOptions::Uninitialized(), m_cellCount * NUM_TETS_PER_HEX, m_cellCount * NUM_TETS_PER_HEX, - m_allocatorId); + m_allocatorId, + m_hostAllocator); AXOM_ANNOTATE_END("allocate m_tets_from_hexes_device"); populateHexesFromMesh(); auto hexesView = m_hexes.view(); AXOM_ANNOTATE_BEGIN("allocate m_cell_volumes"); - m_cell_volumes = axom::Array(m_cellCount, m_cellCount, m_allocatorId); + m_cell_volumes = axom::Array(m_cellCount, m_cellCount, m_allocatorId, m_hostAllocator); AXOM_ANNOTATE_END("allocate m_cell_volumes"); m_cell_volumes.fill(0.0); @@ -532,7 +569,7 @@ class IntersectionShaper : public Shaper axom::fmt::format("{:-^80}", " Decomposing each hexahedron element into 24 tetrahedrons ")); AXOM_ANNOTATE_BEGIN("populate m_hex_bbs"); - m_hex_bbs = axom::Array(m_cellCount, m_cellCount, m_allocatorId); + m_hex_bbs = axom::Array(m_cellCount, m_cellCount, m_allocatorId, m_hostAllocator); // Get bounding boxes for hexahedral elements axom::ArrayView hexBbsView = m_hex_bbs.view(); @@ -561,7 +598,7 @@ class IntersectionShaper : public Shaper AXOM_ANNOTATE_END("init_tets"); AXOM_ANNOTATE_BEGIN("allocate m_overlap_volumes"); - m_overlap_volumes = axom::Array(m_cellCount, m_cellCount, m_allocatorId); + m_overlap_volumes = axom::Array(m_cellCount, m_cellCount, m_allocatorId, m_hostAllocator); AXOM_ANNOTATE_END("allocate m_overlap_volumes"); } @@ -630,19 +667,19 @@ class IntersectionShaper : public Shaper template void prepareTriCells() { - const int host_allocator = axom::execution_space::allocatorID(); + const int host_allocator = m_hostAllocator.getID(); const int device_allocator = axom::execution_space::allocatorID(); // Number of triangles in mesh m_tricount = m_surfaceMesh->getNumberOfCells(); - axom::Array tris_host(m_tricount, m_tricount, host_allocator); + axom::Array tris_host(m_tricount, m_tricount, host_allocator, m_hostAllocator); // Initialize 2D triangles from mesh (ignore z coordinate) - axom::Array nodeIds(3); + axom::Array nodeIds(3, 3, host_allocator, m_hostAllocator); // Buffer is 3D for stl mesh - axom::Array pts(3); + axom::Array pts(3, 3, host_allocator, m_hostAllocator); for(int i = 0; i < m_tricount; i++) { @@ -667,7 +704,7 @@ class IntersectionShaper : public Shaper } // Copy triangles to device - m_tris = axom::Array(tris_host, device_allocator); + m_tris = axom::Array(tris_host, device_allocator, m_hostAllocator); if(this->isVerbose()) { @@ -717,17 +754,17 @@ class IntersectionShaper : public Shaper template void prepareTetCells() { - const int host_allocator = axom::execution_space::allocatorID(); + const int host_allocator = m_hostAllocator.getID(); const int device_allocator = m_allocatorId; // Number of tets in mesh m_tetcount = m_surfaceMesh->getNumberOfCells(); - axom::Array tets_host(m_tetcount, m_tetcount, host_allocator); + axom::Array tets_host(m_tetcount, m_tetcount, host_allocator, m_hostAllocator); // Initialize tetrahedra - axom::Array nodeIds(4); - axom::Array pts(4); + axom::Array nodeIds(4, 4, host_allocator, m_hostAllocator); + axom::Array pts(4, 4, host_allocator, m_hostAllocator); for(int i = 0; i < m_tetcount; i++) { @@ -742,7 +779,7 @@ class IntersectionShaper : public Shaper } // Copy tets to device - m_tets = axom::Array(tets_host, device_allocator); + m_tets = axom::Array(tets_host, device_allocator, m_hostAllocator); if(this->isVerbose()) { @@ -794,12 +831,12 @@ class IntersectionShaper : public Shaper template void prepareC2CCells() { - const int host_allocator = axom::execution_space::allocatorID(); + const int host_allocator = m_hostAllocator.getID(); // Number of points in polyline int pointcount = getSurfaceMesh()->getNumberOfNodes(); - axom::Array polyline(pointcount, pointcount); + axom::Array polyline(pointcount, pointcount, host_allocator, m_hostAllocator); SLIC_INFO(axom::fmt::format( "{:-^80}", @@ -825,9 +862,16 @@ class IntersectionShaper : public Shaper // Generate the Octahedra // (Set m_octs's allocator id to where we want its data to live.) - m_octs = axom::Array(0, 0, axom::execution_space::allocatorID()); - const bool disc_status = - axom::quest::discretize(polyline, polyline_size, m_level, m_octs, m_octcount); + m_octs = axom::Array(0, + 0, + axom::execution_space::allocatorID(), + m_hostAllocator); + const bool disc_status = axom::quest::discretize(polyline, + polyline_size, + m_level, + m_octs, + m_octcount, + m_hostAllocator); axom::ArrayView octs_device_view = m_octs.view(); @@ -843,7 +887,8 @@ class IntersectionShaper : public Shaper { // Print out the bounding box containing all the octahedra BoundingBox3D all_oct_bb; - axom::Array octs_host = axom::Array(m_octs, host_allocator); + axom::Array octs_host = + axom::Array(m_octs, host_allocator, m_hostAllocator); auto octs_host_view = octs_host.view(); for(int i = 0; i < m_octcount; i++) @@ -873,10 +918,10 @@ class IntersectionShaper : public Shaper axom::ReduceSum num_degenerate(0); const int device_allocator = m_allocatorId; - axom::Array degenerate_oct_host(1, 1, host_allocator); + axom::Array degenerate_oct_host(1, 1, host_allocator, m_hostAllocator); degenerate_oct_host[0] = OctahedronType(); axom::Array degenerate_oct_device = - axom::Array(degenerate_oct_host, device_allocator); + axom::Array(degenerate_oct_host, device_allocator, m_hostAllocator); auto degenerate_oct_device_view = degenerate_oct_device.view(); axom::for_all( @@ -917,13 +962,13 @@ class IntersectionShaper : public Shaper // Number of triangles in mesh (2 triangles per segment/quad cell) m_tricount = m_surfaceMesh->getNumberOfCells() * 2; - axom::Array tris_host(m_tricount, m_tricount, host_allocator); + axom::Array tris_host(m_tricount, m_tricount, host_allocator, m_hostAllocator); // Initialize 2D triangles from segment mesh (3rd point is on the x-axis) - axom::Array nodeIds(2); + axom::Array nodeIds(2, 2, host_allocator, m_hostAllocator); // Buffer to store 2D points - axom::Array pts(2); + axom::Array pts(2, 2, host_allocator, m_hostAllocator); for(int i = 0; i < m_tricount / 2; i++) { @@ -947,7 +992,7 @@ class IntersectionShaper : public Shaper } // Copy triangles to device - m_tris = axom::Array(tris_host, device_allocator); + m_tris = axom::Array(tris_host, device_allocator, m_hostAllocator); SLIC_INFO(axom::fmt::format(axom::utilities::locale(), "Contour has been discretized into {:L} triangles ", @@ -1004,14 +1049,15 @@ class IntersectionShaper : public Shaper setMeshDependentData(); } - const int host_allocator = axom::execution_space::allocatorID(); + const int host_allocator = m_hostAllocator.getID(); const int device_allocator = axom::execution_space::allocatorID(); SLIC_INFO(axom::fmt::format("{:-^80}", " Inserting shapes' bounding boxes into BVH ")); // Generate the BVH tree over the shapes // Access-aligned bounding boxes - m_aabbs_2d = axom::Array(shape_count, shape_count, device_allocator); + m_aabbs_2d = + axom::Array(shape_count, shape_count, device_allocator, m_hostAllocator); axom::ArrayView shapes_device_view = shapes.view(); @@ -1044,10 +1090,10 @@ class IntersectionShaper : public Shaper // Find which shape bounding boxes intersect quadrilateral bounding boxes SLIC_INFO(axom::fmt::format("{:-^80}", " Finding shape candidates for each quad element ")); - axom::Array offsets(m_cellCount, m_cellCount, device_allocator); - axom::Array counts(m_cellCount, m_cellCount, device_allocator); + axom::Array offsets(m_cellCount, m_cellCount, device_allocator, m_hostAllocator); + axom::Array counts(m_cellCount, m_cellCount, device_allocator, m_hostAllocator); axom::Array candidates; - bvh.findBoundingBoxes(offsets, counts, candidates, m_cellCount, quad_bbs_device_view); + bvh.findBoundingBoxes(offsets, counts, candidates, m_cellCount, quad_bbs_device_view, m_hostAllocator); // Get the total number of candidates const auto counts_device_view = counts.view(); @@ -1063,7 +1109,8 @@ class IntersectionShaper : public Shaper AXOM_ANNOTATE_BEGIN("allocate quad_indices_device"); axom::Array quad_indices_device(totalCandidates.get(), totalCandidates.get(), - device_allocator); + device_allocator, + m_hostAllocator); AXOM_ANNOTATE_END("allocate quad_indices_device"); auto quad_indices_device_view = quad_indices_device.view(); @@ -1073,17 +1120,18 @@ class IntersectionShaper : public Shaper AXOM_ANNOTATE_BEGIN("allocate shape_candidates_device"); axom::Array shape_candidates_device(totalCandidates.get(), totalCandidates.get(), - device_allocator); + device_allocator, + m_hostAllocator); AXOM_ANNOTATE_END("allocate shape_candidates_device"); auto shape_candidates_device_view = shape_candidates_device.view(); AXOM_ANNOTATE_END("allocate scratch space"); // New total number of candidates after omitting degenerate shapes AXOM_ANNOTATE_BEGIN("newTotalCandidates memory"); - axom::Array newTotalCandidates_host(1, 1, host_allocator); + axom::Array newTotalCandidates_host(1, 1, host_allocator, m_hostAllocator); newTotalCandidates_host[0] = 0; axom::Array newTotalCandidates_device = - axom::Array(newTotalCandidates_host, device_allocator); + axom::Array(newTotalCandidates_host, device_allocator, m_hostAllocator); auto newTotalCandidates_device_view = newTotalCandidates_device.view(); AXOM_ANNOTATE_END("newTotalCandidates memory"); @@ -1122,7 +1170,7 @@ class IntersectionShaper : public Shaper AXOM_ANNOTATE_SCOPE("clipLoop"); // Copy calculated total back to host axom::Array newTotalCandidates_calc_host = - axom::Array(newTotalCandidates_device, host_allocator); + axom::Array(newTotalCandidates_device, host_allocator, m_hostAllocator); axom::for_all( newTotalCandidates_calc_host[0], @@ -1191,7 +1239,7 @@ class IntersectionShaper : public Shaper setMeshDependentData(); } - const int host_allocator = axom::execution_space::allocatorID(); + const int host_allocator = m_hostAllocator.getID(); const int device_allocator = m_allocatorId; constexpr int NUM_TETS_PER_HEX = 24; @@ -1200,7 +1248,7 @@ class IntersectionShaper : public Shaper // Generate the BVH tree over the shapes // Axis-aligned bounding boxes - axom::Array aabbs(shape_count, shape_count, device_allocator); + axom::Array aabbs(shape_count, shape_count, device_allocator, m_hostAllocator); axom::ArrayView shapes_device_view = shapes.view(); @@ -1225,11 +1273,11 @@ class IntersectionShaper : public Shaper SLIC_INFO( axom::fmt::format("{:-^80}", " Finding shape candidates for each hexahedral element ")); - axom::Array offsets(m_cellCount, m_cellCount, device_allocator); - axom::Array counts(m_cellCount, m_cellCount, device_allocator); + axom::Array offsets(m_cellCount, m_cellCount, device_allocator, m_hostAllocator); + axom::Array counts(m_cellCount, m_cellCount, device_allocator, m_hostAllocator); axom::Array candidates; AXOM_ANNOTATE_BEGIN("bvh.findBoundingBoxes"); - bvh.findBoundingBoxes(offsets, counts, candidates, m_cellCount, hex_bbs_device_view); + bvh.findBoundingBoxes(offsets, counts, candidates, m_cellCount, hex_bbs_device_view, m_hostAllocator); AXOM_ANNOTATE_END("bvh.findBoundingBoxes"); // Get the total number of candidates @@ -1246,14 +1294,16 @@ class IntersectionShaper : public Shaper AXOM_ANNOTATE_BEGIN("allocate hex_indices"); axom::Array hex_indices_device(totalCandidates.get() * NUM_TETS_PER_HEX, totalCandidates.get() * NUM_TETS_PER_HEX, - device_allocator); + device_allocator, + m_hostAllocator); AXOM_ANNOTATE_END("allocate hex_indices"); auto hex_indices_device_view = hex_indices_device.view(); AXOM_ANNOTATE_BEGIN("allocate shape_candidates"); axom::Array shape_candidates_device(totalCandidates.get() * NUM_TETS_PER_HEX, totalCandidates.get() * NUM_TETS_PER_HEX, - device_allocator); + device_allocator, + m_hostAllocator); AXOM_ANNOTATE_END("allocate shape_candidates"); auto shape_candidates_device_view = shape_candidates_device.view(); @@ -1264,17 +1314,18 @@ class IntersectionShaper : public Shaper AXOM_ANNOTATE_BEGIN("allocate tet_indices_device"); axom::Array tet_indices_device(totalCandidates.get() * NUM_TETS_PER_HEX, totalCandidates.get() * NUM_TETS_PER_HEX, - device_allocator); + device_allocator, + m_hostAllocator); AXOM_ANNOTATE_END("allocate tet_indices_device"); auto tet_indices_device_view = tet_indices_device.view(); AXOM_ANNOTATE_END("allocate scratch space"); // New total number of candidates after omitting degenerate shapes AXOM_ANNOTATE_BEGIN("newTotalCandidates memory"); - axom::Array newTotalCandidates_host(1, 1, host_allocator); + axom::Array newTotalCandidates_host(1, 1, host_allocator, m_hostAllocator); newTotalCandidates_host[0] = 0; axom::Array newTotalCandidates_device = - axom::Array(newTotalCandidates_host, device_allocator); + axom::Array(newTotalCandidates_host, device_allocator, m_hostAllocator); auto newTotalCandidates_device_view = newTotalCandidates_device.view(); AXOM_ANNOTATE_END("newTotalCandidates memory"); @@ -1319,7 +1370,7 @@ class IntersectionShaper : public Shaper AXOM_ANNOTATE_SCOPE("clipLoop"); // Copy calculated total back to host axom::Array newTotalCandidates_calc_host = - axom::Array(newTotalCandidates_device, host_allocator); + axom::Array(newTotalCandidates_device, host_allocator, m_hostAllocator); axom::for_all( newTotalCandidates_calc_host[0], // Number of candidates found. @@ -1523,8 +1574,8 @@ class IntersectionShaper : public Shaper " IntersectionShaper::applyReplacementRules."); // Allocate some memory for the replacement rule data arrays. - Array vf_subtract_array(dataSize, dataSize, m_allocatorId); - Array vf_writable_array(dataSize, dataSize, m_allocatorId); + Array vf_subtract_array(dataSize, dataSize, m_allocatorId, m_hostAllocator); + Array vf_writable_array(dataSize, dataSize, m_allocatorId, m_hostAllocator); ArrayView vf_subtract(vf_subtract_array); ArrayView vf_writable(vf_writable_array); @@ -2622,7 +2673,8 @@ class IntersectionShaper : public Shaper axom::Array vertCoords(m_cellCount * NUM_VERTS_PER_QUAD * NUM_COMPS_PER_VERT, m_cellCount * NUM_VERTS_PER_QUAD * NUM_COMPS_PER_VERT, - allocId); + allocId, + m_hostAllocator); #if defined(AXOM_USE_MFEM) if(m_dc != nullptr) @@ -2640,7 +2692,8 @@ class IntersectionShaper : public Shaper auto vertCoords_device_view = vertCoords.view(); // Initialize quad elements - m_quads = axom::Array(m_cellCount, m_cellCount, m_allocatorId); + m_quads = + axom::Array(m_cellCount, m_cellCount, m_allocatorId, m_hostAllocator); axom::ArrayView quads_device_view = m_quads.view(); axom::for_all( @@ -2668,7 +2721,8 @@ class IntersectionShaper : public Shaper axom::Array vertCoords(m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, - allocId); + allocId, + m_hostAllocator); #if defined(AXOM_USE_MFEM) if(m_dc != nullptr) @@ -2685,7 +2739,7 @@ class IntersectionShaper : public Shaper auto vertCoords_device_view = vertCoords.view(); - m_hexes = axom::Array(m_cellCount, m_cellCount, allocId); + m_hexes = axom::Array(m_cellCount, m_cellCount, allocId, m_hostAllocator); axom::ArrayView hexes_device_view = m_hexes.view(); axom::for_all( m_cellCount, @@ -2753,7 +2807,8 @@ class IntersectionShaper : public Shaper vertCoords = axom::Array(m_cellCount * NUM_VERTS_PER_QUAD * NUM_COMPS_PER_VERT, m_cellCount * NUM_VERTS_PER_QUAD * NUM_COMPS_PER_VERT, - allocId); + allocId, + m_hostAllocator); auto vertCoordsView = vertCoords.view(); axom::for_all( @@ -2768,8 +2823,9 @@ class IntersectionShaper : public Shaper auto vertId = quadVerts[j]; for(int k = 0; k < NUM_COMPS_PER_VERT; k++) { - vertCoordsView[(i * NUM_VERTS_PER_QUAD * NUM_COMPS_PER_VERT) + (j * NUM_COMPS_PER_VERT) + k] = - coordArrays[k][vertId]; + const auto coordIdx = + (i * NUM_VERTS_PER_QUAD * NUM_COMPS_PER_VERT) + (j * NUM_COMPS_PER_VERT) + k; + vertCoordsView[coordIdx] = coordArrays[k][vertId]; } } }); @@ -2827,7 +2883,8 @@ class IntersectionShaper : public Shaper vertCoords = axom::Array(m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, - allocId); + allocId, + m_hostAllocator); auto vertCoordsView = vertCoords.view(); axom::for_all( @@ -2842,8 +2899,9 @@ class IntersectionShaper : public Shaper auto vertId = hexVerts[j]; for(int k = 0; k < NUM_COMPS_PER_VERT; k++) { - vertCoordsView[(i * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT) + (j * NUM_COMPS_PER_VERT) + k] = - coordArrays[k][vertId]; + const auto coordIdx = + (i * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT) + (j * NUM_COMPS_PER_VERT) + k; + vertCoordsView[coordIdx] = coordArrays[k][vertId]; } } }); @@ -2891,8 +2949,12 @@ class IntersectionShaper : public Shaper axom::Array& fillVertCoords = axom::execution_space::onDevice() ? tmpVertCoords : vertCoords; + const int fillAllocId = + axom::execution_space::onDevice() ? m_hostAllocator.getID() : m_allocatorId; fillVertCoords = axom::Array(m_cellCount * num_verts_per_cell * num_comps_per_vert, - m_cellCount * num_verts_per_cell * num_comps_per_vert); + m_cellCount * num_verts_per_cell * num_comps_per_vert, + fillAllocId, + m_hostAllocator); // Initialize vertices from mfem mesh and // set each shape volume fraction to 1 @@ -2990,6 +3052,7 @@ class IntersectionShaper : public Shaper private: int m_level {DEFAULT_CIRCLE_REFINEMENT_LEVEL}; double m_revolvedVolume {DEFAULT_REVOLVED_VOLUME}; + HostAllocator m_hostAllocator; std::string m_free_mat_name; //! \brief Volumes of cells in the computational mesh. diff --git a/src/axom/quest/MarchingCubes.cpp b/src/axom/quest/MarchingCubes.cpp index 4a0d4f855e..379ae31a49 100644 --- a/src/axom/quest/MarchingCubes.cpp +++ b/src/axom/quest/MarchingCubes.cpp @@ -27,8 +27,16 @@ const axom::StackArray twoZeros {0, 0}; MarchingCubes::MarchingCubes(RuntimePolicy runtimePolicy, int allocatorID, MarchingCubesDataParallelism dataParallelism) + : MarchingCubes(runtimePolicy, allocatorID, HostAllocator {}, dataParallelism) +{ } + +MarchingCubes::MarchingCubes(RuntimePolicy runtimePolicy, + int allocatorID, + HostAllocator hostAllocator, + MarchingCubesDataParallelism dataParallelism) : m_runtimePolicy(runtimePolicy) , m_allocatorID(allocatorID) + , m_hostAllocator(hostAllocator) , m_dataParallelism(dataParallelism) , m_singles() , m_topologyName() @@ -38,14 +46,14 @@ MarchingCubes::MarchingCubes(RuntimePolicy runtimePolicy, , m_maskPath() , m_facetIndexOffsets(0, 0) , m_facetCount(0) - , m_caseIdsFlat(0, 0, m_allocatorID) - , m_crossingFlags(0, 0, m_allocatorID) - , m_scannedFlags(0, 0, m_allocatorID) - , m_facetIncrs(0, 0, m_allocatorID) - , m_facetNodeIds(twoZeros, m_allocatorID) - , m_facetNodeCoords(twoZeros, m_allocatorID) - , m_facetParentIds(0, 0, m_allocatorID) - , m_facetDomainIds(0, 0, m_allocatorID) + , m_caseIdsFlat(0, 0, m_allocatorID, m_hostAllocator) + , m_crossingFlags(0, 0, m_allocatorID, m_hostAllocator) + , m_scannedFlags(0, 0, m_allocatorID, m_hostAllocator) + , m_facetIncrs(0, 0, m_allocatorID, m_hostAllocator) + , m_facetNodeIds(twoZeros, m_allocatorID, m_hostAllocator) + , m_facetNodeCoords(twoZeros, m_allocatorID, m_hostAllocator) + , m_facetParentIds(0, 0, m_allocatorID, m_hostAllocator) + , m_facetDomainIds(0, 0, m_allocatorID, m_hostAllocator) { } // Set the object up for a blueprint mesh state. @@ -195,19 +203,13 @@ void MarchingCubes::populateContourMesh(axom::mint::UnstructuredMesh() -#else - axom::detail::getAllocatorID() -#endif - : m_allocatorID; + const int hostAllocatorId = + hostAndInternalMemoriesAreSeparate ? m_hostAllocator.getID() : m_allocatorID; if(hostAndInternalMemoriesAreSeparate) { - axom::Array tmpfacetNodeCoords(m_facetNodeCoords, hostAllocatorId); - axom::Array tmpfacetNodeIds(m_facetNodeIds, hostAllocatorId); + axom::Array tmpfacetNodeCoords(m_facetNodeCoords, hostAllocatorId, m_hostAllocator); + axom::Array tmpfacetNodeIds(m_facetNodeIds, hostAllocatorId, m_hostAllocator); mesh.appendNodes(tmpfacetNodeCoords.data(), contourNodeCount); mesh.appendCells(tmpfacetNodeIds.data(), contourCellCount); } diff --git a/src/axom/quest/MarchingCubes.hpp b/src/axom/quest/MarchingCubes.hpp index 4b663a0cbe..015d238292 100644 --- a/src/axom/quest/MarchingCubes.hpp +++ b/src/axom/quest/MarchingCubes.hpp @@ -19,6 +19,7 @@ #ifdef AXOM_USE_CONDUIT // Axom includes + #include "axom/core/memory_management.hpp" #include "axom/core/execution/runtime_policy.hpp" #include "axom/mint/mesh/UnstructuredMesh.hpp" @@ -118,12 +119,20 @@ class MarchingCubes * running sequentially on the CPU. * @param [in] allocatorID Data allocator ID. Choose something compatible * with \c runtimePolicy. See \c execution_space. + * @param [in] hostAllocator Host allocator used for host-resident scratch + * and staging allocations. Defaults to Axom's legacy host + * allocator convenience path. * @param [in] dataParallelism Data parallel implementation choice. */ MarchingCubes(RuntimePolicy runtimePolicy, int allocatorId, MarchingCubesDataParallelism dataParallelism); + MarchingCubes(RuntimePolicy runtimePolicy, + int allocatorId, + HostAllocator hostAllocator, + MarchingCubesDataParallelism dataParallelism); + /*! * @brief Set the input mesh. * @param [in] bpMesh Blueprint multi-domain mesh containing scalar field. @@ -294,6 +303,7 @@ class MarchingCubes private: RuntimePolicy m_runtimePolicy; int m_allocatorID = axom::INVALID_ALLOCATOR_ID; + HostAllocator m_hostAllocator {}; //! @brief Choice of full or partial data-parallelism, or byPolicy. MarchingCubesDataParallelism m_dataParallelism = MarchingCubesDataParallelism::byPolicy; diff --git a/src/axom/quest/MeshClipper.cpp b/src/axom/quest/MeshClipper.cpp index 003c0154ba..3c0852b1ce 100644 --- a/src/axom/quest/MeshClipper.cpp +++ b/src/axom/quest/MeshClipper.cpp @@ -40,13 +40,15 @@ MeshClipper::MeshClipper(quest::experimental::ShapeMesh& shapeMesh, void MeshClipper::clip(axom::Array& ovlap) { const int allocId = m_shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = m_shapeMesh.getHostAllocator(); const axom::IndexType cellCount = m_shapeMesh.getCellCount(); // Resize output array and use appropriate allocator. if(ovlap.size() < cellCount || ovlap.getAllocatorID() != allocId) { AXOM_ANNOTATE_SCOPE("MeshClipper:clip_alloc"); - ovlap = axom::Array(ArrayOptions::Uninitialized(), cellCount, cellCount, allocId); + ovlap = + axom::Array(ArrayOptions::Uninitialized(), cellCount, cellCount, allocId, hostAllocator); } clip(ovlap.view()); } @@ -244,9 +246,9 @@ std::unique_ptr MeshClipper::newImpl() #if defined(AXOM_USE_MPI) template -void globalReduce(axom::Array& values, MPI_Op reduceOp) +void globalReduce(axom::Array& values, MPI_Op reduceOp, HostAllocator hostAllocator) { - axom::Array localValues(values); + axom::Array localValues(values, values.getAllocatorID(), hostAllocator); MPI_Allreduce(localValues.data(), values.data(), values.size(), @@ -292,15 +294,19 @@ conduit::Node MeshClipper::getGlobalClippingStats() const #if defined(AXOM_USE_MPI) // Do sum and max reductions. - axom::Array sums(0, sumNode.number_of_children()); + HostAllocator hostAllocator = m_shapeMesh.getHostAllocator(); + axom::Array sums(0, + sumNode.number_of_children(), + hostAllocator.getID(), + hostAllocator); for(int i = 0; i < sumNode.number_of_children(); ++i) { const axom::IndexType value = locNode.child(i).value(); sums.push_back(value); } - axom::Array maxs(sums); - globalReduce(maxs, MPI_MAX); - globalReduce(sums, MPI_SUM); + axom::Array maxs(sums, hostAllocator.getID(), hostAllocator); + globalReduce(maxs, MPI_MAX, hostAllocator); + globalReduce(sums, MPI_SUM, hostAllocator); for(int i = 0; i < sumNode.number_of_children(); ++i) { diff --git a/src/axom/quest/MeshTester.hpp b/src/axom/quest/MeshTester.hpp index 4c0f5cd0cf..9128efc591 100644 --- a/src/axom/quest/MeshTester.hpp +++ b/src/axom/quest/MeshTester.hpp @@ -56,6 +56,8 @@ enum class WatertightStatus : signed char * \param [out] degenerateIndices indices of degenerate mesh triangles * \param [in] intersectionThreshold Tolerance threshold for triangle * intersection tests (default: 1E-8) + * \param [in] hostAllocator Allocator to use for host-accessible scratch and + * staging * After running this function over a surface mesh, intersection will be filled * with pairs of indices of intersecting triangles and degenerateIndices will * be filled with the indices of the degenerate triangles in the mesh. @@ -67,7 +69,8 @@ template void findTriMeshIntersectionsBVH(mint::UnstructuredMesh* surface_mesh, std::vector>& intersections, std::vector& degenerateIndices, - double intersectionThreshold = 1E-8) + double intersectionThreshold = 1E-8, + HostAllocator hostAllocator = HostAllocator {}) { AXOM_ANNOTATE_SCOPE("quest::findTriMeshIntersectionsBVH"); @@ -76,7 +79,7 @@ void findTriMeshIntersectionsBVH(mint::UnstructuredMesh* sur constexpr detail::AccelType UseBVH = detail::AccelType::BVH; using CandidateFinder = detail::CandidateFinder; - CandidateFinder impl(surface_mesh, intersectionThreshold); + CandidateFinder impl(surface_mesh, intersectionThreshold, hostAllocator); impl.initialize(); axom::Array intersectFirst, intersectSecond, degenerate; impl.findTriMeshIntersections(intersectFirst, intersectSecond, degenerate); @@ -100,6 +103,8 @@ void findTriMeshIntersectionsBVH(mint::UnstructuredMesh* sur * structure (default: 0) * \param [in] intersectionThreshold Tolerance threshold for triangle * intersection tests (default: 1E-8) + * \param [in] hostAllocator Allocator to use for host-accessible scratch and + * staging * After running this function over a surface mesh, intersection will be filled * with pairs of indices of intersecting triangles and degenerateIndices will * be filled with the indices of the degenerate triangles in the mesh. @@ -117,7 +122,8 @@ void findTriMeshIntersectionsImplicitGrid(mint::UnstructuredMesh>& intersections, std::vector& degenerateIndices, int spatialIndexResolution = 0, - double intersectionThreshold = 1E-8) + double intersectionThreshold = 1E-8, + HostAllocator hostAllocator = HostAllocator {}) { AXOM_ANNOTATE_SCOPE("quest::findTriMeshIntersectionsImplicitGrid"); @@ -126,7 +132,7 @@ void findTriMeshIntersectionsImplicitGrid(mint::UnstructuredMesh; - CandidateFinder impl(surface_mesh, intersectionThreshold); + CandidateFinder impl(surface_mesh, intersectionThreshold, hostAllocator); impl.initialize(spatialIndexResolution); axom::Array intersectFirst, intersectSecond, degenerate; impl.findTriMeshIntersections(intersectFirst, intersectSecond, degenerate); @@ -150,6 +156,8 @@ void findTriMeshIntersectionsImplicitGrid(mint::UnstructuredMesh>& intersections, std::vector& degenerateIndices, int spatialIndexResolution = 0, - double intersectionThreshold = 1E-8) + double intersectionThreshold = 1E-8, + HostAllocator hostAllocator = HostAllocator {}) { AXOM_ANNOTATE_SCOPE("quest::findTriMeshIntersectionsUniformGrid"); @@ -176,7 +185,7 @@ void findTriMeshIntersectionsUniformGrid(mint::UnstructuredMesh; - CandidateFinder impl(surface_mesh, intersectionThreshold); + CandidateFinder impl(surface_mesh, intersectionThreshold, hostAllocator); impl.initialize(spatialIndexResolution); axom::Array intersectFirst, intersectSecond, degenerate; impl.findTriMeshIntersections(intersectFirst, intersectSecond, degenerate); diff --git a/src/axom/quest/PointInCell.hpp b/src/axom/quest/PointInCell.hpp index 5023596c7f..62f1b1d454 100644 --- a/src/axom/quest/PointInCell.hpp +++ b/src/axom/quest/PointInCell.hpp @@ -7,6 +7,7 @@ #pragma once #include "axom/config.hpp" +#include "axom/core/memory_management.hpp" #include "axom/core/Macros.hpp" #include "axom/slic.hpp" @@ -110,6 +111,9 @@ class PointInCell * the bounding boxes. Default: 1e-8 * \param [in] allocatorId Currently unused. Default value is based on the * allocator ID set for the specified execution space. + * \param [in] hostAllocator Host allocator used for host-resident bounding + * boxes and host-side staging buffers. Defaults to Axom's legacy host + * allocator convenience path. * * \note The bboxTolerance should be a small positive number. It helps avoid * numerical issues in the bounding box containment queries by slightly @@ -126,7 +130,8 @@ class PointInCell PointInCell(MeshType* mesh, int* resolution = nullptr, double bboxTolerance = 1e-8, - int allocatorID = axom::execution_space::allocatorID()) + int allocatorID = axom::execution_space::allocatorID(), + HostAllocator hostAllocator = HostAllocator {}) : m_meshWrapper(mesh) , m_pointFinder2D(nullptr) , m_pointFinder3D(nullptr) @@ -140,10 +145,12 @@ class PointInCell switch(m_meshWrapper.meshDimension()) { case 2: - m_pointFinder2D = new PointFinder2D(&m_meshWrapper, resolution, bboxScaleFactor, allocatorID); + m_pointFinder2D = + new PointFinder2D(&m_meshWrapper, resolution, bboxScaleFactor, allocatorID, hostAllocator); break; case 3: - m_pointFinder3D = new PointFinder3D(&m_meshWrapper, resolution, bboxScaleFactor, allocatorID); + m_pointFinder3D = + new PointFinder3D(&m_meshWrapper, resolution, bboxScaleFactor, allocatorID, hostAllocator); break; default: SLIC_ERROR("Point in Cell query only defined for 2D or 3D meshes."); diff --git a/src/axom/quest/SamplingShaper.hpp b/src/axom/quest/SamplingShaper.hpp index e2b6d6c8bb..a21c2571c1 100644 --- a/src/axom/quest/SamplingShaper.hpp +++ b/src/axom/quest/SamplingShaper.hpp @@ -143,6 +143,21 @@ class SamplingShaper : public Shaper } } + SamplingShaper(RuntimePolicy execPolicy, + int allocatorId, + HostAllocator hostAllocator, + const klee::ShapeSet& shapeSet, + sidre::MFEMSidreDataCollection* dc) + : Shaper(execPolicy, allocatorId, hostAllocator, shapeSet, dc) + { + // Initialize the default number of samples based on the mesh dimension. + const int dim = getMeshDimension(); + for(int d = 0; d < dim; d++) + { + m_samplingResolution.push_back(5); + } + } + ~SamplingShaper() { m_inoutShapeQFuncs.DeleteData(true); @@ -374,7 +389,8 @@ class SamplingShaper : public Shaper const auto format = this->shapeFormat(shape); if(useWindingNumberSampler(shape)) { - m_sampler = std::make_unique(shapeName, m_contours.view()); + m_sampler = + std::make_unique(shapeName, m_contours.view(), m_hostAllocator); } else if(format == "c2c" || format == "mfem") { @@ -390,21 +406,25 @@ class SamplingShaper : public Shaper switch(this->getExecutionPolicy()) { case Policy::seq: - m_sampler = std::make_unique(shapeName, m_surfaceMesh); + m_sampler = + std::make_unique(shapeName, m_surfaceMesh, m_hostAllocator); break; #if defined(AXOM_RUNTIME_POLICY_USE_OPENMP) case Policy::omp: - m_sampler = std::make_unique(shapeName, m_surfaceMesh); + m_sampler = + std::make_unique(shapeName, m_surfaceMesh, m_hostAllocator); break; #endif #if defined(AXOM_RUNTIME_POLICY_USE_CUDA) case Policy::cuda: - m_sampler = std::make_unique(shapeName, m_surfaceMesh); + m_sampler = + std::make_unique(shapeName, m_surfaceMesh, m_hostAllocator); break; #endif #if defined(AXOM_RUNTIME_POLICY_USE_HIP) case Policy::hip: - m_sampler = std::make_unique(shapeName, m_surfaceMesh); + m_sampler = + std::make_unique(shapeName, m_surfaceMesh, m_hostAllocator); break; #endif default: diff --git a/src/axom/quest/ScatteredInterpolation.hpp b/src/axom/quest/ScatteredInterpolation.hpp index 1308858317..ffbd324e96 100644 --- a/src/axom/quest/ScatteredInterpolation.hpp +++ b/src/axom/quest/ScatteredInterpolation.hpp @@ -316,7 +316,7 @@ class ScatteredInterpolation const int npts = pts.size(); if(npts <= 1) { - axom::Array reordered(npts, npts); + axom::Array reordered(npts, npts, m_hostAllocator.getID(), m_hostAllocator); for(int idx = 0; idx < npts; ++idx) { reordered[idx] = idx; @@ -439,7 +439,7 @@ class ScatteredInterpolation } // Extract and return the reordered point indices. - axom::Array reordered(npts, npts); + axom::Array reordered(npts, npts, m_hostAllocator.getID(), m_hostAllocator); for(int idx = 0; idx < npts; ++idx) { reordered[idx] = bucketed[static_cast(idx)].second; @@ -449,6 +449,19 @@ class ScatteredInterpolation } public: + /*! + * \brief Constructs a scattered interpolation object. + * + * \param [in] hostAllocator allocator used for host-resident BRIO ordering data. + */ + explicit ScatteredInterpolation(HostAllocator hostAllocator = HostAllocator {}) + : m_hostAllocator(hostAllocator) + , m_brio_data(0, 0, hostAllocator.getID(), hostAllocator) + { } + + /// Returns the host allocator used for host-resident ordering data. + HostAllocator getHostAllocator() const { return m_hostAllocator; } + /** * \brief Builds a Delaunay triangulation over the point set from \a mesh_node * @@ -742,6 +755,7 @@ class ScatteredInterpolation private: DelaunayTriangulation m_delaunay; + HostAllocator m_hostAllocator; axom::Array m_brio_data; VertexIndirectionSet m_brio; BoundingBoxType m_bounding_box; diff --git a/src/axom/quest/ShapeMesh.cpp b/src/axom/quest/ShapeMesh.cpp index 71d01ab264..2bdcd8770f 100644 --- a/src/axom/quest/ShapeMesh.cpp +++ b/src/axom/quest/ShapeMesh.cpp @@ -30,10 +30,20 @@ ShapeMesh::ShapeMesh(RuntimePolicy runtimePolicy, conduit::Node& bpMesh, const std::string& topoName, const std::string& matsetName) + : ShapeMesh(runtimePolicy, allocatorId, HostAllocator {}, bpMesh, topoName, matsetName) +{ } + +ShapeMesh::ShapeMesh(RuntimePolicy runtimePolicy, + int allocatorId, + HostAllocator hostAllocator, + conduit::Node& bpMesh, + const std::string& topoName, + const std::string& matsetName) : m_runtimePolicy(runtimePolicy) , m_allocId(allocatorId != axom::INVALID_ALLOCATOR_ID ? allocatorId : axom::policyToDefaultAllocatorID(runtimePolicy)) + , m_hostAllocator(hostAllocator) , m_topoName(topoName.empty() && bpMesh["topologies"].number_of_children() > 0 ? bpMesh["topologies"].child(0).name() : topoName) @@ -47,8 +57,6 @@ ShapeMesh::ShapeMesh(RuntimePolicy runtimePolicy, SLIC_ERROR_IF(m_topoName.empty(), "Topology name was not provided, and no default topology was found."); - const int hostAllocId = axom::execution_space::allocatorID(); - // We currently support only unstructured topo. const auto& typeNode = m_bpNodeExt->fetch_existing("topologies").fetch_existing(m_topoName).fetch_existing("type"); @@ -69,7 +77,7 @@ ShapeMesh::ShapeMesh(RuntimePolicy runtimePolicy, // If matsetName was given, but topology data isn't set up yet, set it up. if(!matsetNode.has_child("topology")) { - matsetNode.set_allocator(sidre::ConduitMemory::axomAllocIdToConduit(hostAllocId)); + matsetNode.set_allocator(sidre::ConduitMemory::axomAllocIdToConduit(m_hostAllocator.getID())); matsetNode.fetch("topology").set_string(m_topoName); } @@ -107,10 +115,20 @@ ShapeMesh::ShapeMesh(RuntimePolicy runtimePolicy, sidre::Group* bpMesh, const std::string& topoName, const std::string& matsetName) + : ShapeMesh(runtimePolicy, allocatorId, HostAllocator {}, bpMesh, topoName, matsetName) +{ } + +ShapeMesh::ShapeMesh(RuntimePolicy runtimePolicy, + int allocatorId, + HostAllocator hostAllocator, + sidre::Group* bpMesh, + const std::string& topoName, + const std::string& matsetName) : m_runtimePolicy(runtimePolicy) , m_allocId(allocatorId != axom::INVALID_ALLOCATOR_ID ? allocatorId : axom::policyToDefaultAllocatorID(runtimePolicy)) + , m_hostAllocator(hostAllocator) , m_topoName(topoName.empty() && bpMesh->hasGroup("topologies") && bpMesh->getGroup("topologies")->getNumGroups() > 0 ? bpMesh->getGroup("topologies")->getGroup(0)->getName() @@ -780,8 +798,11 @@ void ShapeMesh::computeCellsAsHexesImpl() axom::ArrayView connView = getCellNodeConnectivity(); - m_cellsAsHexes = - axom::Array(ArrayOptions::Uninitialized(), m_cellCount, m_cellCount, m_allocId); + m_cellsAsHexes = axom::Array(ArrayOptions::Uninitialized(), + m_cellCount, + m_cellCount, + m_allocId, + m_hostAllocator); axom::ArrayView cellsAsHexesView = m_cellsAsHexes.view(); SLIC_ASSERT(cellsAsHexesView.data() == m_cellsAsHexes.data()); @@ -819,7 +840,8 @@ void ShapeMesh::computeCellsAsTetsImpl() m_cellsAsTets = axom::Array(ArrayOptions::Uninitialized(), NUM_TETS_PER_HEX * m_cellCount, NUM_TETS_PER_HEX * m_cellCount, - m_allocId); + m_allocId, + m_hostAllocator); auto cellsAsTetsView = m_cellsAsTets.view(); auto cellsAsHexesView = getCellsAsHexes(); @@ -836,8 +858,11 @@ void ShapeMesh::computeCellsAsTetsImpl() template void ShapeMesh::computeHexVolumesImpl() { - m_hexVolumes = - axom::Array(ArrayOptions::Uninitialized(), m_cellCount, m_cellCount, m_allocId); + m_hexVolumes = axom::Array(ArrayOptions::Uninitialized(), + m_cellCount, + m_cellCount, + m_allocId, + m_hostAllocator); auto cellsAsHexes = getCellsAsHexes(); @@ -851,7 +876,8 @@ template void ShapeMesh::computeTetVolumesImpl() { axom::IndexType tetCount = m_cellCount * NUM_TETS_PER_HEX; - m_tetVolumes = axom::Array(ArrayOptions::Uninitialized(), tetCount, tetCount, m_allocId); + m_tetVolumes = + axom::Array(ArrayOptions::Uninitialized(), tetCount, tetCount, m_allocId, m_hostAllocator); auto cellsAsTets = getCellsAsTets(); @@ -864,8 +890,11 @@ void ShapeMesh::computeTetVolumesImpl() template void ShapeMesh::computeHexBbsImpl() { - m_hexBbs = - axom::Array(ArrayOptions::Uninitialized(), m_cellCount, m_cellCount, m_allocId); + m_hexBbs = axom::Array(ArrayOptions::Uninitialized(), + m_cellCount, + m_cellCount, + m_allocId, + m_hostAllocator); auto cellsAsHexes = getCellsAsHexes(); @@ -880,8 +909,11 @@ void ShapeMesh::computeHexBbsImpl() template void ShapeMesh::computeCellLengthsImpl() { - m_cellLengths = - axom::Array(ArrayOptions::Uninitialized(), m_cellCount, m_cellCount, m_allocId); + m_cellLengths = axom::Array(ArrayOptions::Uninitialized(), + m_cellCount, + m_cellCount, + m_allocId, + m_hostAllocator); auto cellBbs = getCellBoundingBoxes(); @@ -894,7 +926,7 @@ void ShapeMesh::computeCellLengthsImpl() template void ShapeMesh::computeVertPointsImpl() { - m_vertPoints3D = axom::Array(m_vertexCount, m_vertexCount, m_allocId); + m_vertPoints3D = axom::Array(m_vertexCount, m_vertexCount, m_allocId, m_hostAllocator); auto& vertCoords = getVertexCoords3D(); const auto& vX = vertCoords[0]; diff --git a/src/axom/quest/ShapeMesh.hpp b/src/axom/quest/ShapeMesh.hpp index e91be8272a..62410b5a94 100644 --- a/src/axom/quest/ShapeMesh.hpp +++ b/src/axom/quest/ShapeMesh.hpp @@ -103,6 +103,13 @@ class ShapeMesh const std::string& topoName = {}, const std::string& matsetName = {}); + ShapeMesh(RuntimePolicy runtimePolicy, + int allocatorId, + HostAllocator hostAllocator, + conduit::Node& bpMesh, + const std::string& topoName = {}, + const std::string& matsetName = {}); + #ifdef AXOM_USE_SIDRE /*! * @brief Constructor with computational mesh in a sidre::Group. @@ -129,6 +136,13 @@ class ShapeMesh sidre::Group* bpMesh, const std::string& topoName = {}, const std::string& matsetName = {}); + + ShapeMesh(RuntimePolicy runtimePolicy, + int allocatorId, + HostAllocator hostAllocator, + sidre::Group* bpMesh, + const std::string& topoName = {}, + const std::string& matsetName = {}); #endif /*! @@ -141,6 +155,11 @@ class ShapeMesh */ int getAllocatorID() const { return m_allocId; } + /*! + * @brief Host allocator set in constructor. + */ + HostAllocator getHostAllocator() const { return m_hostAllocator; } + /* * !@brief Return computational mesh as a sidre::Group if it has * that form, or nullptr otherwise. @@ -263,6 +282,7 @@ class ShapeMesh const RuntimePolicy m_runtimePolicy; int m_allocId; + HostAllocator m_hostAllocator; //! @brief Mesh topology name. const std::string m_topoName; diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index 5e7b005ec5..8f47914655 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -33,10 +33,19 @@ Shaper::Shaper(RuntimePolicy execPolicy, int allocatorId, const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) + : Shaper(execPolicy, allocatorId, HostAllocator {}, shapeSet, dc) +{ } + +Shaper::Shaper(RuntimePolicy execPolicy, + int allocatorId, + HostAllocator hostAllocator, + const klee::ShapeSet& shapeSet, + sidre::MFEMSidreDataCollection* dc) : m_execPolicy(execPolicy) , m_allocatorId(allocatorId != axom::INVALID_ALLOCATOR_ID ? allocatorId : axom::policyToDefaultAllocatorID(execPolicy)) + , m_hostAllocator(hostAllocator) , m_shapeSet(shapeSet) , m_dc(dc) #if defined(AXOM_USE_CONDUIT) @@ -60,10 +69,20 @@ Shaper::Shaper(RuntimePolicy execPolicy, const klee::ShapeSet& shapeSet, sidre::Group* bpGrp, const std::string& topo) + : Shaper(execPolicy, allocatorId, HostAllocator {}, shapeSet, bpGrp, topo) +{ } + +Shaper::Shaper(RuntimePolicy execPolicy, + int allocatorId, + HostAllocator hostAllocator, + const klee::ShapeSet& shapeSet, + sidre::Group* bpGrp, + const std::string& topo) : m_execPolicy(execPolicy) , m_allocatorId(allocatorId != axom::INVALID_ALLOCATOR_ID ? allocatorId : axom::policyToDefaultAllocatorID(execPolicy)) + , m_hostAllocator(hostAllocator) , m_shapeSet(shapeSet) #if defined(AXOM_USE_CONDUIT) , m_bpGrp(bpGrp) @@ -91,10 +110,20 @@ Shaper::Shaper(RuntimePolicy execPolicy, const klee::ShapeSet& shapeSet, conduit::Node& bpNode, const std::string& topo) + : Shaper(execPolicy, allocatorId, HostAllocator {}, shapeSet, bpNode, topo) +{ } + +Shaper::Shaper(RuntimePolicy execPolicy, + int allocatorId, + HostAllocator hostAllocator, + const klee::ShapeSet& shapeSet, + conduit::Node& bpNode, + const std::string& topo) : m_execPolicy(execPolicy) , m_allocatorId(allocatorId != axom::INVALID_ALLOCATOR_ID ? allocatorId : axom::policyToDefaultAllocatorID(execPolicy)) + , m_hostAllocator(hostAllocator) , m_shapeSet(shapeSet) #if defined(AXOM_USE_CONDUIT) , m_bpGrp(nullptr) @@ -245,7 +274,7 @@ void Shaper::loadShapeInternal(const klee::Shape& shape, double percentError, do axom::fmt::format("Shape has unsupported format: '{}", this->shapeFormat(shape))); // Code for discretizing shapes has been factored into DiscreteShape class. - DiscreteShape discreteShape(shape, m_dataStore.getRoot(), m_prefixPath); + DiscreteShape discreteShape(shape, m_dataStore.getRoot(), m_hostAllocator, m_prefixPath); discreteShape.setSamplesPerKnotSpan(m_samplesPerKnotSpan); discreteShape.setVertexWeldThreshold(m_vertexWeldThreshold); discreteShape.setRefinementType(m_refinementType); diff --git a/src/axom/quest/Shaper.hpp b/src/axom/quest/Shaper.hpp index caadea8300..020138db72 100644 --- a/src/axom/quest/Shaper.hpp +++ b/src/axom/quest/Shaper.hpp @@ -22,6 +22,7 @@ #endif #include "axom/sidre.hpp" +#include "axom/core/memory_management.hpp" #include "axom/klee.hpp" #include "axom/mint.hpp" #include "axom/quest/DiscreteShape.hpp" @@ -56,6 +57,12 @@ class Shaper int allocatorId, const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc); + + Shaper(RuntimePolicy execPolicy, + int allocatorId, + HostAllocator hostAllocator, + const klee::ShapeSet& shapeSet, + sidre::MFEMSidreDataCollection* dc); #endif /*! @@ -68,6 +75,13 @@ class Shaper sidre::Group* bpMesh, const std::string& topo = ""); + Shaper(RuntimePolicy execPolicy, + int allocatorId, + HostAllocator hostAllocator, + const klee::ShapeSet& shapeSet, + sidre::Group* bpMesh, + const std::string& topo = ""); + /*! * @brief Construct Shaper to operate on a blueprint-formatted mesh * stored in a conduit Node. @@ -83,6 +97,13 @@ class Shaper conduit::Node& bpNode, const std::string& topo = ""); + Shaper(RuntimePolicy execPolicy, + int allocatorId, + HostAllocator hostAllocator, + const klee::ShapeSet& shapeSet, + conduit::Node& bpNode, + const std::string& topo = ""); + virtual ~Shaper(); public: @@ -233,6 +254,7 @@ class Shaper protected: RuntimePolicy m_execPolicy; int m_allocatorId; + HostAllocator m_hostAllocator; // For any mesh represented in Conduit or sidre sidre::DataStore m_dataStore; diff --git a/src/axom/quest/detail/Discretize_detail.hpp b/src/axom/quest/detail/Discretize_detail.hpp index 5d0b158174..2824a40fb1 100644 --- a/src/axom/quest/detail/Discretize_detail.hpp +++ b/src/axom/quest/detail/Discretize_detail.hpp @@ -148,10 +148,13 @@ inline OctType new_inscribed_prism(OctType &old_oct, * quadrilateral side-wall. */ template -int discrSeg(const Point2D &a, const Point2D &b, int levels, axom::ArrayView &out, int idx) +int discrSeg(const Point2D &a, + const Point2D &b, + int levels, + axom::ArrayView &out, + int idx, + axom::HostAllocator hostAllocator) { - int hostAllocID = axom::execution_space::allocatorID(); - // Assert input assumptions SLIC_ASSERT(a[1] >= 0); SLIC_ASSERT(b[1] >= 0); @@ -171,7 +174,7 @@ int discrSeg(const Point2D &a, const Point2D &b, int levels, axom::ArrayView(1, hostAllocID); + OctType *oct_from_seg = axom::allocate(1, hostAllocator.getID()); oct_from_seg[0] = from_segment(a, b); axom::copy(out.data() + idx + 0, oct_from_seg, sizeof(OctType)); @@ -263,7 +266,8 @@ bool discretize(const axom::ArrayView &polyline, int pointcount, int levels, axom::Array &out, - int &octcount) + int &octcount, + HostAllocator hostAllocator) { SLIC_ERROR_IF(!axom::execution_space::usesAllocId(out.getAllocatorID()), axom::fmt::format("Execution space {} cannot access allocator id {}", @@ -300,7 +304,7 @@ bool discretize(const axom::ArrayView &polyline, for(int seg = 0; seg < segmentcount; ++seg) { int segment_prism_count = - discrSeg(polyline[seg], polyline[seg + 1], levels, out_view, octcount); + discrSeg(polyline[seg], polyline[seg + 1], levels, out_view, octcount, hostAllocator); octcount += segment_prism_count; } // octcount may be < totaloctcount if there are degenerate segments. diff --git a/src/axom/quest/detail/DistributedClosestPointImpl.hpp b/src/axom/quest/detail/DistributedClosestPointImpl.hpp index d932a92583..6b201cdd18 100644 --- a/src/axom/quest/detail/DistributedClosestPointImpl.hpp +++ b/src/axom/quest/detail/DistributedClosestPointImpl.hpp @@ -243,8 +243,9 @@ inline int isend_using_schema(conduit::Node& node, class DistributedClosestPointImpl { public: - DistributedClosestPointImpl(int allocatorID, bool isVerbose) + DistributedClosestPointImpl(int allocatorID, HostAllocator hostAllocator, bool isVerbose) : m_allocatorID(allocatorID) + , m_hostAllocator(hostAllocator) , m_isVerbose(isVerbose) , m_mpiComm(MPI_COMM_NULL) , m_rank(-1) @@ -266,6 +267,8 @@ class DistributedClosestPointImpl m_allocatorID = allocatorID; } + void setHostAllocator(HostAllocator hostAllocator) { m_hostAllocator = hostAllocator; } + /*! @brief Import object mesh points from the object blueprint mesh into internal memory. @@ -516,6 +519,7 @@ class DistributedClosestPointImpl protected: int m_allocatorID; + HostAllocator m_hostAllocator; bool m_isVerbose; MPI_Comm m_mpiComm; @@ -575,10 +579,10 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl Also see setAllocatorID(). @param [i[ isVerbose */ - DistributedClosestPointExec(int allocatorID, bool isVerbose) - : DistributedClosestPointImpl(allocatorID, isVerbose) - , m_objectPtCoords(0, 0, allocatorID) - , m_objectPtDomainIds(0, 0, allocatorID) + DistributedClosestPointExec(int allocatorID, HostAllocator hostAllocator, bool isVerbose) + : DistributedClosestPointImpl(allocatorID, hostAllocator, isVerbose) + , m_objectPtCoords(0, 0, allocatorID, hostAllocator) + , m_objectPtDomainIds(0, 0, allocatorID, hostAllocator) { SLIC_ASSERT(allocatorID != axom::INVALID_ALLOCATOR_ID); @@ -606,8 +610,8 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl } // Copy points to internal memory - PointArray coords(ptCount, ptCount); - axom::Array domIds(ptCount, ptCount); + PointArray coords(ptCount, ptCount, m_hostAllocator.getID(), m_hostAllocator); + axom::Array domIds(ptCount, ptCount, m_hostAllocator.getID(), m_hostAllocator); std::size_t copiedCount = 0; conduit::Node tmpValues; for(axom::IndexType d = 0; d < mdMeshNode.number_of_children(); ++d) @@ -644,8 +648,8 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl copiedCount += N; } // copy computed data to ExecSpace - m_objectPtCoords = PointArray(coords, m_allocatorID); - m_objectPtDomainIds = axom::Array(domIds, m_allocatorID); + m_objectPtCoords = PointArray(coords, m_allocatorID, m_hostAllocator); + m_objectPtDomainIds = axom::Array(domIds, m_allocatorID, m_hostAllocator); } bool generateBVHTree() override @@ -659,7 +663,7 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl // move the object point data to avoid repetitive page faults. if(m_objectPtCoords.getAllocatorID() != m_allocatorID) { - PointArray tmpPoints(m_objectPtCoords, m_allocatorID); + PointArray tmpPoints(m_objectPtCoords, m_allocatorID, m_hostAllocator); m_objectPtCoords.swap(tmpPoints); } @@ -679,10 +683,13 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl /// Allgather one bounding box from each rank. void gatherBoundingBoxes(const BoxType& aabb, BoxArray& all_aabbs) const { - axom::Array sendbuf(2 * DIM); + axom::Array sendbuf(2 * DIM, 2 * DIM, m_hostAllocator.getID(), m_hostAllocator); aabb.getMin().to_array(&sendbuf[0]); aabb.getMax().to_array(&sendbuf[DIM]); - axom::Array recvbuf(m_nranks * sendbuf.size()); + axom::Array recvbuf(m_nranks * sendbuf.size(), + m_nranks * sendbuf.size(), + m_hostAllocator.getID(), + m_hostAllocator); // Note: Using axom::Array may reduce clutter a tad. int errf = MPI_Allgather(sendbuf.data(), 2 * DIM, @@ -885,7 +892,7 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl SLIC_ASSERT(bvh != nullptr); const int npts = m_objectPtCoords.size(); - axom::Array boxesArray(npts, npts, m_allocatorID); + axom::Array boxesArray(npts, npts, m_allocatorID, m_hostAllocator); auto boxesView = boxesArray.view(); auto pointsView = m_objectPtCoords.view(); @@ -939,17 +946,21 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl /// Create ArrayViews in ExecSpace that are compatible with fields // This deep-copies host memory in xferDom to device memory. // TODO: Avoid copying arrays (here and at the end) if both are on the host - auto cp_idx = is_first ? axom::Array(qPtCount, qPtCount, m_allocatorID) - : axom::Array(cpIndexes, m_allocatorID); - auto cp_domidx = is_first ? axom::Array(qPtCount, qPtCount, m_allocatorID) - : axom::Array(cpDomainIndexes, m_allocatorID); - auto cp_rank = is_first ? axom::Array(qPtCount, qPtCount, m_allocatorID) - : axom::Array(cpRanks, m_allocatorID); + auto cp_idx = is_first + ? axom::Array(qPtCount, qPtCount, m_allocatorID, m_hostAllocator) + : axom::Array(cpIndexes, m_allocatorID, m_hostAllocator); + auto cp_domidx = is_first + ? axom::Array(qPtCount, qPtCount, m_allocatorID, m_hostAllocator) + : axom::Array(cpDomainIndexes, m_allocatorID, m_hostAllocator); + auto cp_rank = is_first + ? axom::Array(qPtCount, qPtCount, m_allocatorID, m_hostAllocator) + : axom::Array(cpRanks, m_allocatorID, m_hostAllocator); /// PROBLEM: The striding does not appear to be retained by conduit relay /// We might need to transform it? or to use a single array w/ pointers into it? - auto cp_pos = is_first ? axom::Array(qPtCount, qPtCount, m_allocatorID) - : axom::Array(cpCoords, m_allocatorID); + auto cp_pos = is_first + ? axom::Array(qPtCount, qPtCount, m_allocatorID, m_hostAllocator) + : axom::Array(cpCoords, m_allocatorID, m_hostAllocator); // DEBUG const bool has_cp_distance = xferDom.has_path("debug/cp_distance"); @@ -958,9 +969,9 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl : ArrayView(); auto cp_dist = has_cp_distance - ? (is_first ? axom::Array(qPtCount, qPtCount, m_allocatorID) - : axom::Array(minDist, m_allocatorID)) - : axom::Array(0, 0, m_allocatorID); + ? (is_first ? axom::Array(qPtCount, qPtCount, m_allocatorID, m_hostAllocator) + : axom::Array(minDist, m_allocatorID, m_hostAllocator)) + : axom::Array(0, 0, m_allocatorID, m_hostAllocator); // END DEBUG if(is_first) @@ -979,7 +990,7 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl auto query_min_dist = cp_dist.view(); /// Create an ArrayView in ExecSpace that is compatible with queryPts - PointArray execPoints(queryPts, m_allocatorID); + PointArray execPoints(queryPts, m_allocatorID, m_hostAllocator); auto query_pts = execPoints.view(); if(hasObjectPoints) @@ -988,12 +999,10 @@ class DistributedClosestPointExec : public DistributedClosestPointImpl auto it = m_bvh->getTraverser(); const int rank = m_rank; - axom::Array sqDistThresh_host(1, - 1, - axom::execution_space::allocatorID()); + axom::Array sqDistThresh_host(1, 1, m_hostAllocator.getID(), m_hostAllocator); sqDistThresh_host[0] = m_sqDistanceThreshold; axom::Array sqDistThresh_device = - axom::Array(sqDistThresh_host, m_allocatorID); + axom::Array(sqDistThresh_host, m_allocatorID, m_hostAllocator); auto sqDistThresh_device_view = sqDistThresh_device.view(); auto ptCoordsView = m_objectPtCoords.view(); diff --git a/src/axom/quest/detail/MeshTester_detail.hpp b/src/axom/quest/detail/MeshTester_detail.hpp index bcf7d93284..b65008da50 100644 --- a/src/axom/quest/detail/MeshTester_detail.hpp +++ b/src/axom/quest/detail/MeshTester_detail.hpp @@ -81,8 +81,8 @@ struct CandidateFinderBase { using BoxType = typename primal::BoundingBox; using PointType = typename primal::Point; -#ifdef AXOM_USE_UMPIRE static constexpr bool ExecOnDevice = axom::execution_space::onDevice(); +#ifdef AXOM_USE_UMPIRE static constexpr MemorySpace Space = ExecOnDevice ? axom::MemorySpace::Device : axom::MemorySpace::Host; static constexpr MemorySpace HostSpace = axom::MemorySpace::Host; @@ -99,9 +99,15 @@ struct CandidateFinderBase * triangle intersection tests. */ CandidateFinderBase(mint::UnstructuredMesh* surface_mesh, - double intersectionThreshold) + double intersectionThreshold, + HostAllocator hostAllocator = HostAllocator {}) : m_surfaceMesh(surface_mesh) , m_intersectionThreshold(intersectionThreshold) + , m_hostAllocator(hostAllocator) + , m_allocatorId(ExecOnDevice ? axom::detail::getAllocatorID() : m_hostAllocator.getID()) + , m_tris(0, 0, m_allocatorId, m_hostAllocator) + , m_aabbs(0, 0, m_allocatorId, m_hostAllocator) + , m_degenerate(0, 0, m_allocatorId, m_hostAllocator) { } /*! @@ -144,6 +150,8 @@ struct CandidateFinderBase mint::UnstructuredMesh* m_surfaceMesh; double m_intersectionThreshold; + HostAllocator m_hostAllocator; + int m_allocatorId; int m_ncells; axom::Array m_tris; axom::Array m_aabbs; @@ -206,18 +214,19 @@ void CandidateFinderBase::findTriMeshIntersections( using HostIndexArray = axom::Array; // Get CSR arrays for candidate data - IndexArray offsets, counts; + IndexArray offsets(0, 0, m_allocatorId, m_hostAllocator); + IndexArray counts(0, 0, m_allocatorId, m_hostAllocator); IndexView candidates = getCandidates(offsets, counts); - IndexArray indices(candidates.size()); - IndexArray validCandidates(candidates.size()); + IndexArray indices(candidates.size(), candidates.size(), m_allocatorId, m_hostAllocator); + IndexArray validCandidates(candidates.size(), candidates.size(), m_allocatorId, m_hostAllocator); auto v_indices = indices.view(); auto v_validCandidates = validCandidates.view(); IndexType numCandidates; { - IndexArray numValidCandidates(1); + IndexArray numValidCandidates(1, 1, m_allocatorId, m_hostAllocator); numValidCandidates.fill(0); auto v_numValidCandidates = numValidCandidates.view(); @@ -242,11 +251,11 @@ void CandidateFinderBase::findTriMeshIntersections( axom::copy(&numCandidates, numValidCandidates.data(), sizeof(IndexType)); } - IndexArray firstIsectPair(candidates.size()); - IndexArray secondIsectPair(candidates.size()); + IndexArray firstIsectPair(candidates.size(), candidates.size(), m_allocatorId, m_hostAllocator); + IndexArray secondIsectPair(candidates.size(), candidates.size(), m_allocatorId, m_hostAllocator); IndexType isectCounter; { - IndexArray numIsectPairs(1); + IndexArray numIsectPairs(1, 1, m_allocatorId, m_hostAllocator); auto v_numIsectPairs = numIsectPairs.view(); auto v_firstIsectPair = firstIsectPair.view(); @@ -276,9 +285,9 @@ void CandidateFinderBase::findTriMeshIntersections( { // copy results to output on host - firstIndex = HostIndexArray(firstIsectPair); - secondIndex = HostIndexArray(secondIsectPair); - HostIndexArray host_degenerate = m_degenerate; + firstIndex = axom::Array(firstIsectPair, m_hostAllocator.getID(), m_hostAllocator); + secondIndex = axom::Array(secondIsectPair, m_hostAllocator.getID(), m_hostAllocator); + HostIndexArray host_degenerate(m_degenerate, m_hostAllocator); for(int i = 0; i < host_degenerate.size(); i++) { if(host_degenerate[i] == 1) @@ -306,16 +315,20 @@ struct CandidateFinder axom::Array& offsets, axom::Array& counts) override { - int allocatorId = axom::detail::getAllocatorID(); spin::BVH<3, ExecSpace, FloatType> bvh; - bvh.setAllocatorID(allocatorId); + bvh.setAllocatorID(this->m_allocatorId); bvh.initialize(this->m_aabbs.view(), this->m_aabbs.size()); offsets.resize(this->m_aabbs.size()); counts.resize(this->m_aabbs.size()); // Search for intersecting bounding boxes of triangles - bvh.findBoundingBoxes(offsets, counts, m_currCandidates, this->m_aabbs.size(), this->m_aabbs.view()); + bvh.findBoundingBoxes(offsets, + counts, + m_currCandidates, + this->m_aabbs.size(), + this->m_aabbs.view(), + this->m_hostAllocator); return m_currCandidates; } @@ -387,17 +400,20 @@ struct CandidateFinder axom::Array& offsets, axom::Array& counts) override { - int allocatorId = axom::detail::getAllocatorID(); axom::spin::ImplicitGrid<3, ExecSpace, IndexType> gridIndex(m_globalBox, &m_resolutions, this->m_aabbs.size(), - allocatorId); + this->m_allocatorId); gridIndex.insert(this->m_aabbs.size(), this->m_aabbs.data()); offsets.resize(this->m_aabbs.size()); counts.resize(this->m_aabbs.size()); - gridIndex.getCandidatesAsArray(this->m_aabbs, offsets, counts, m_currCandidates); + gridIndex.getCandidatesAsArray(this->m_aabbs, + offsets, + counts, + m_currCandidates, + this->m_hostAllocator); return m_currCandidates; } @@ -439,9 +455,10 @@ struct CandidateFinder axom::Array& offsets, axom::Array& counts) override { - int allocatorId = axom::detail::getAllocatorID(); - - axom::Array indices(this->m_aabbs.size()); + axom::Array indices(this->m_aabbs.size(), + this->m_aabbs.size(), + this->m_allocatorId, + this->m_hostAllocator); const auto indices_v = indices.view(); for_all(this->m_aabbs.size(), AXOM_LAMBDA(IndexType idx) { indices_v[idx] = idx; }); @@ -450,12 +467,16 @@ struct CandidateFinder spin::UniformGrid gridIndex(m_resolutions, this->m_aabbs.view(), indices.view(), - allocatorId); + this->m_allocatorId); offsets.resize(this->m_aabbs.size()); counts.resize(this->m_aabbs.size()); - gridIndex.getCandidatesAsArray(this->m_aabbs, offsets, counts, m_currCandidates); + gridIndex.getCandidatesAsArray(this->m_aabbs, + offsets, + counts, + m_currCandidates, + this->m_hostAllocator); return m_currCandidates; } diff --git a/src/axom/quest/detail/PointFinder.hpp b/src/axom/quest/detail/PointFinder.hpp index 1329f0e946..58542c178b 100644 --- a/src/axom/quest/detail/PointFinder.hpp +++ b/src/axom/quest/detail/PointFinder.hpp @@ -6,6 +6,7 @@ #pragma once +#include "axom/core/memory_management.hpp" #include "axom/spin/ImplicitGrid.hpp" #include "axom/primal/geometry/BoundingBox.hpp" @@ -64,9 +65,14 @@ class PointFinder * * \sa constructors in PointInCell class for more details about parameters */ - PointFinder(const MeshWrapperType* meshWrapper, const int* res, double bboxScaleFactor, int allocatorID) + PointFinder(const MeshWrapperType* meshWrapper, + const int* res, + double bboxScaleFactor, + int allocatorID, + HostAllocator hostAllocator) : m_meshWrapper(meshWrapper) , m_allocatorID(allocatorID) + , m_hostAllocator(hostAllocator) { SLIC_ASSERT(m_meshWrapper != nullptr); SLIC_ASSERT(bboxScaleFactor >= 1.); @@ -76,18 +82,17 @@ class PointFinder // setup bounding boxes -- Slightly scaled for robustness SpatialBoundingBox meshBBox; -#ifdef AXOM_USE_UMPIRE - axom::Array cellBBoxesHost(numCells); -#else - axom::Array cellBBoxesHost(numCells); -#endif + axom::Array cellBBoxesHost(numCells, + numCells, + m_hostAllocator.getID(), + m_hostAllocator); m_meshWrapper->template computeBoundingBoxes(bboxScaleFactor, cellBBoxesHost.data(), meshBBox); if(DeviceExec) { // Copy the host-side bounding boxes to GPU memory. - m_cellBBoxes = axom::Array(cellBBoxesHost, m_allocatorID); + m_cellBBoxes = axom::Array(cellBBoxesHost, m_allocatorID, m_hostAllocator); } else { @@ -125,7 +130,9 @@ class PointFinder if(DeviceExec) { - axom::Array dev_ptr(axom::ArrayView(&pt, 1), m_allocatorID); + axom::Array dev_ptr(axom::ArrayView(&pt, 1), + m_allocatorID, + m_hostAllocator); locatePoints(dev_ptr, &containingCell, &isopar); } else @@ -150,29 +157,20 @@ class PointFinder #ifdef AXOM_USE_RAJA using IndexView = axom::ArrayView; - #ifdef AXOM_USE_UMPIRE - using HostIndexArray = axom::Array; - using HostPointArray = axom::Array; - - using HostIndexView = axom::ArrayView; - using HostPointView = axom::ArrayView; - using ConstHostPointView = axom::ArrayView; - #else - using HostIndexArray = IndexArray; + using HostIndexArray = axom::Array; using HostPointArray = axom::Array; - using HostIndexView = IndexView; + using HostIndexView = axom::ArrayView; using HostPointView = axom::ArrayView; using ConstHostPointView = axom::ArrayView; - #endif // AXOM_USE_UMPIRE -#endif // AXOM_USE_RAJA +#endif // AXOM_USE_RAJA auto gridQuery = m_grid.getQueryObject(); axom::IndexType npts = pts.size(); - IndexArray offsets(npts, npts, m_allocatorID); - IndexArray counts(npts, npts, m_allocatorID); + IndexArray offsets(npts, npts, m_allocatorID, m_hostAllocator); + IndexArray counts(npts, npts, m_allocatorID, m_hostAllocator); #ifdef AXOM_USE_RAJA IndexView countsPtr = counts; @@ -192,7 +190,7 @@ class PointFinder axom::IndexType totalCount = totalCountReduce.get(); // Step 3: allocate memory for all candidates - IndexArray candidates(totalCount, totalCount, m_allocatorID); + IndexArray candidates(totalCount, totalCount, m_allocatorID, m_hostAllocator); IndexView candidatesPtr = candidates; IndexView offsetsPtr = offsets; const SpatialBoundingBox* cellBBoxes = m_cellBBoxes.data(); @@ -218,87 +216,70 @@ class PointFinder countsPtr[i] = currCount; }); - // Temporary host arrays we copy device-side data into when the candidate - // search is conducted on the GPU - HostPointArray ptsHost, outIsoparHost; - HostIndexArray outCellIdsHost; - HostIndexArray candidatesHost, offsetsHost, countsHost; - - // For sequential/OpenMP execution, just use the argument pointers - // directly. - HostIndexView outCellIdsPtr(outCellIds, pts.size()); - HostPointView outIsoparPtr(outIsoparametricCoords, pts.size()); - - // If the candidate search takes place on the GPU, we need to copy the - // device-side data first, then set these array views to point to the - // intermediate arrays. Otherwise, we can set these to point to the result - // arrays directly. - ConstHostPointView ptsHostPtr; - HostIndexView candidatesHostPtr, offsetsHostPtr, countsHostPtr; + auto locateCandidatesHost = [&](ConstHostPointView ptsHostPtr, + HostIndexView candidatesHostPtr, + HostIndexView offsetsHostPtr, + HostIndexView countsHostPtr, + HostIndexView outCellIdsPtr, + HostPointView outIsoparPtr) { + // Step 5: Check each candidate + // TODO: This only supports sequential execution right now, because we + // don't build MFEM in a thread-safe manner. + const MeshWrapperType* meshWrapperPtr = m_meshWrapper; + for_all( + npts, + AXOM_HOST_LAMBDA(IndexType i) { + outCellIdsPtr[i] = PointInCellTraits::NO_CELL; + const SpacePoint& pt = ptsHostPtr[i]; + SpacePoint isopar; + for(int icell = 0; icell < countsHostPtr[i]; icell++) + { + const int cellIdx = candidatesHostPtr[icell + offsetsHostPtr[i]]; + if(meshWrapperPtr->locatePointInCell(cellIdx, pt.data(), isopar.data())) + { + outCellIdsPtr[i] = cellIdx; + break; + } + } + if(outIsoparametricCoords != nullptr) + { + outIsoparPtr[i] = isopar; + } + }); + }; if(DeviceExec) { - // Copy points and candidate intersections to host memory. - ptsHost = pts; - candidatesHost = candidates; - offsetsHost = offsets; - countsHost = counts; - // Set up views from intermediate host arrays - ptsHostPtr = ptsHost; - candidatesHostPtr = candidatesHost; - offsetsHostPtr = offsetsHost; - countsHostPtr = countsHost; - // Allocate intermediate output buffers on the host side. - outCellIdsHost.resize(pts.size()); - if(outIsoparametricCoords) + HostPointArray ptsHost(pts, m_hostAllocator.getID(), m_hostAllocator); + HostIndexArray candidatesHost(candidates, m_hostAllocator.getID(), m_hostAllocator); + HostIndexArray offsetsHost(offsets, m_hostAllocator.getID(), m_hostAllocator); + HostIndexArray countsHost(counts, m_hostAllocator.getID(), m_hostAllocator); + HostIndexArray outCellIdsHost(npts, npts, m_hostAllocator.getID(), m_hostAllocator); + HostPointArray outIsoparHost(0, 0, m_hostAllocator.getID(), m_hostAllocator); + if(outIsoparametricCoords != nullptr) { - outIsoparHost.resize(pts.size()); + outIsoparHost.resize(npts); } - outCellIdsPtr = outCellIdsHost; - outIsoparPtr = outIsoparHost; - } - else - { - ptsHostPtr = pts; - candidatesHostPtr = candidates; - offsetsHostPtr = offsets; - countsHostPtr = counts; - } - // Step 5: Check each candidate - // TODO: This only supports sequential execution right now, because we - // don't build MFEM in a thread-safe manner. - const MeshWrapperType* meshWrapperPtr = m_meshWrapper; - for_all( - npts, - AXOM_HOST_LAMBDA(IndexType i) { - outCellIdsPtr[i] = PointInCellTraits::NO_CELL; - const SpacePoint& pt = ptsHostPtr[i]; - SpacePoint isopar; - for(int icell = 0; icell < countsHostPtr[i]; icell++) - { - const int cellIdx = candidatesHostPtr[icell + offsetsHostPtr[i]]; - // if isopar is in the proper range - if(meshWrapperPtr->locatePointInCell(cellIdx, pt.data(), isopar.data())) - { - // then we have found the cellID - outCellIdsPtr[i] = cellIdx; - break; - } - } - if(outIsoparametricCoords != nullptr) - { - outIsoparPtr[i] = isopar; - } - }); + locateCandidatesHost(ptsHost, candidatesHost, offsetsHost, countsHost, outCellIdsHost, outIsoparHost); - if(DeviceExec) - { // Copy back to GPU memory. axom::copy(outCellIds, outCellIdsHost.data(), outCellIdsHost.size() * sizeof(IndexType)); - axom::copy(outIsoparametricCoords, - outIsoparHost.data(), - outIsoparHost.size() * sizeof(SpacePoint)); + if(outIsoparametricCoords != nullptr) + { + axom::copy(outIsoparametricCoords, + outIsoparHost.data(), + outIsoparHost.size() * sizeof(SpacePoint)); + } + } + else + { + locateCandidatesHost(pts, + candidates, + offsets, + counts, + HostIndexView(outCellIds, pts.size()), + HostPointView(outIsoparametricCoords, pts.size())); } #else // AXOM_USE_RAJA for(int i = 0; i < npts; i++) @@ -341,6 +322,7 @@ class PointFinder const MeshWrapperType* m_meshWrapper; axom::Array m_cellBBoxes; int m_allocatorID; + HostAllocator m_hostAllocator; }; } // end namespace detail diff --git a/src/axom/quest/detail/clipping/HexClipper.cpp b/src/axom/quest/detail/clipping/HexClipper.cpp index abc0b9cf00..71b858b4c5 100644 --- a/src/axom/quest/detail/clipping/HexClipper.cpp +++ b/src/axom/quest/detail/clipping/HexClipper.cpp @@ -53,10 +53,15 @@ bool HexClipper::labelCellsInOut(quest::experimental::ShapeMesh& shapeMesh, SLIC_ERROR_IF(shapeMesh.dimension() != 3, "HexClipper requires a 3D mesh."); int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); auto cellCount = shapeMesh.getCellCount(); if(labels.size() < cellCount || labels.getAllocatorID() != shapeMesh.getAllocatorID()) { - labels = axom::Array(ArrayOptions::Uninitialized(), cellCount, cellCount, allocId); + labels = axom::Array(ArrayOptions::Uninitialized(), + cellCount, + cellCount, + allocId, + hostAllocator); } switch(shapeMesh.getRuntimePolicy()) @@ -91,6 +96,7 @@ bool HexClipper::labelTetsInOut(quest::experimental::ShapeMesh& shapeMesh, { const axom::IndexType cellCount = cellIds.size(); const int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); if(tetLabels.size() < cellCount * NUM_TETS_PER_HEX || tetLabels.getAllocatorID() != shapeMesh.getAllocatorID()) @@ -98,7 +104,8 @@ bool HexClipper::labelTetsInOut(quest::experimental::ShapeMesh& shapeMesh, tetLabels = axom::Array(ArrayOptions::Uninitialized(), cellCount * NUM_TETS_PER_HEX, cellCount * NUM_TETS_PER_HEX, - allocId); + allocId, + hostAllocator); } switch(shapeMesh.getRuntimePolicy()) @@ -133,12 +140,13 @@ void HexClipper::labelCellsInOutImpl(quest::experimental::ShapeMesh& shapeMesh, { const auto cellCount = shapeMesh.getCellCount(); const int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); const auto cellBbs = shapeMesh.getCellBoundingBoxes(); const auto cellsAsHexes = shapeMesh.getCellsAsHexes(); const auto cellVolumes = shapeMesh.getCellVolumes(); const auto hexBb = m_hexBb; const auto surfaceTriangles = m_surfaceTriangles; - axom::Array tets(m_tets, allocId); + axom::Array tets(m_tets, allocId, hostAllocator); axom::ArrayView tetsView = tets.view(); constexpr double EPS = 1e-10; @@ -166,11 +174,12 @@ void HexClipper::labelTetsInOutImpl(quest::experimental::ShapeMesh& shapeMesh, { const axom::IndexType cellCount = cellIds.size(); const int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); auto meshHexes = shapeMesh.getCellsAsHexes(); auto tetVolumes = shapeMesh.getTetVolumes(); const auto hexBb = m_hexBb; const auto surfaceTriangles = m_surfaceTriangles; - axom::Array tets(m_tets, allocId); + axom::Array tets(m_tets, allocId, hostAllocator); axom::ArrayView tetsView = tets.view(); constexpr double EPS = 1e-10; @@ -251,9 +260,10 @@ bool HexClipper::getGeometryAsTets(quest::experimental::ShapeMesh& shapeMesh, axom::Array& tets) { int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); if(tets.getAllocatorID() != allocId || tets.size() != m_tets.size()) { - tets = axom::Array(m_tets.size(), m_tets.size(), allocId); + tets = axom::Array(m_tets.size(), m_tets.size(), allocId, hostAllocator); } axom::copy(tets.data(), m_tets.data(), m_tets.size() * sizeof(TetrahedronType)); return true; diff --git a/src/axom/quest/detail/clipping/MeshClipperImpl.hpp b/src/axom/quest/detail/clipping/MeshClipperImpl.hpp index 622e6aa290..a41b1c91be 100644 --- a/src/axom/quest/detail/clipping/MeshClipperImpl.hpp +++ b/src/axom/quest/detail/clipping/MeshClipperImpl.hpp @@ -111,11 +111,12 @@ class MeshClipperImpl : public MeshClipper::Impl void collectOnIndices(const axom::ArrayView& labels, axom::Array& onIndices) override { + HostAllocator hostAllocator = getShapeMesh().getHostAllocator(); if(labels.empty()) { if(onIndices.getAllocatorID() != labels.getAllocatorID()) { - onIndices = axom::Array(0, 0, labels.getAllocatorID()); + onIndices = axom::Array(0, 0, labels.getAllocatorID(), hostAllocator); } return; }; @@ -137,7 +138,8 @@ class MeshClipperImpl : public MeshClipper::Impl axom::Array tmpLabels(ArrayOptions::Uninitialized(), 1 + labels.size(), 0, - labels.getAllocatorID()); + labels.getAllocatorID(), + hostAllocator); tmpLabels.fill(0, 1, 0); auto tmpLabelsView = tmpLabels.view(); axom::ReduceSum onCountReduce {0}; @@ -158,7 +160,8 @@ class MeshClipperImpl : public MeshClipper::Impl onIndices = axom::Array {axom::ArrayOptions::Uninitialized(), onCount, 0, - labels.getAllocatorID()}; + labels.getAllocatorID(), + hostAllocator}; } auto onIndicesView = onIndices.view(); @@ -223,11 +226,13 @@ class MeshClipperImpl : public MeshClipper::Impl void computeClipVolumes3D(axom::ArrayView ovlap, conduit::Node& statistics) override { ShapeMesh& shapeMesh = getShapeMesh(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); const IndexType tetCount = shapeMesh.getCellCount() * ShapeMesh::NUM_TETS_PER_HEX; axom::Array tetIndices(ArrayOptions::Uninitialized(), tetCount, 0, - shapeMesh.getAllocatorID()); + shapeMesh.getAllocatorID(), + hostAllocator); auto tetIndicesView = tetIndices.view(); axom::for_all(tetCount, AXOM_LAMBDA(IndexType ti) { tetIndicesView[ti] = ti; }); computeClipVolumes3DTets(tetIndicesView, ovlap, statistics); @@ -245,12 +250,14 @@ class MeshClipperImpl : public MeshClipper::Impl { ShapeMesh& shapeMesh = getShapeMesh(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); const IndexType cellCount = cellIndices.size(); const IndexType tetCount = cellCount * ShapeMesh::NUM_TETS_PER_HEX; axom::Array tetIndices(ArrayOptions::Uninitialized(), tetCount, 0, - shapeMesh.getAllocatorID()); + shapeMesh.getAllocatorID(), + hostAllocator); auto tetIndicesView = tetIndices.view(); axom::for_all( cellCount, @@ -283,6 +290,7 @@ class MeshClipperImpl : public MeshClipper::Impl auto meshTets = getShapeMesh().getCellsAsTets(); const int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); /* * Geometry as discrete tets or octs, and their bounding boxes. @@ -304,7 +312,7 @@ class MeshClipperImpl : public MeshClipper::Impl // containing only those listed in tetIndices. // The BVH searches on this array. const axom::IndexType tetCount = tetIndices.size(); - axom::Array tetBbs(tetCount, tetCount, allocId); + axom::Array tetBbs(tetCount, tetCount, allocId, hostAllocator); axom::ArrayView tetBbsView = tetBbs.view(); axom::for_all( tetCount, @@ -315,8 +323,8 @@ class MeshClipperImpl : public MeshClipper::Impl for(int j = 0; j < 4; ++j) tetBb.addPoint(tet[j]); }); - axom::Array counts(tetCount, tetCount, allocId); - axom::Array offsets(tetCount, tetCount, allocId); + axom::Array counts(tetCount, tetCount, allocId, hostAllocator); + axom::Array offsets(tetCount, tetCount, allocId, hostAllocator); axom::Array candidates; auto countsView = counts.view(); auto offsetsView = offsets.view(); @@ -379,8 +387,8 @@ class MeshClipperImpl : public MeshClipper::Impl * - candToTetIdId: indicates the meshTets in the collision, * where candToTetIdId[i] corresponds to meshTets[tetIndices[i]]. */ - candidates = axom::Array(nCollisions, nCollisions, allocId); - axom::Array candToTetIdId(candidates.size(), candidates.size(), allocId); + candidates = axom::Array(nCollisions, nCollisions, allocId, hostAllocator); + axom::Array candToTetIdId(candidates.size(), candidates.size(), allocId, hostAllocator); auto candidatesView = candidates.view(); auto candToTetIdIdView = candToTetIdId.view(); @@ -498,6 +506,7 @@ class MeshClipperImpl : public MeshClipper::Impl auto& strategy = getStrategy(); ShapeMesh& shapeMesh = getShapeMesh(); int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); AXOM_ANNOTATE_BEGIN("MeshClipper:get_geometry"); const bool useOcts = strategy.getGeometryAsOcts(shapeMesh, geomAsOcts); @@ -532,7 +541,7 @@ class MeshClipperImpl : public MeshClipper::Impl * Get the bounding boxes for the discrete geometry pieces. */ const axom::IndexType bbCount = useTets ? geomAsTets.size() : geomAsOcts.size(); - pieceBbs = axom::Array(bbCount, bbCount, allocId); + pieceBbs = axom::Array(bbCount, bbCount, allocId, hostAllocator); axom::ArrayView pieceBbsView = pieceBbs.view(); if(useTets) diff --git a/src/axom/quest/detail/clipping/MonotonicZSORClipper.cpp b/src/axom/quest/detail/clipping/MonotonicZSORClipper.cpp index 1e3ff965a5..48c75372c4 100644 --- a/src/axom/quest/detail/clipping/MonotonicZSORClipper.cpp +++ b/src/axom/quest/detail/clipping/MonotonicZSORClipper.cpp @@ -23,6 +23,12 @@ namespace experimental { MonotonicZSORClipper::MonotonicZSORClipper(const klee::Geometry& kGeom, const std::string& name) + : MonotonicZSORClipper(kGeom, name, HostAllocator {}) +{ } + +MonotonicZSORClipper::MonotonicZSORClipper(const klee::Geometry& kGeom, + const std::string& name, + HostAllocator hostAllocator) : MeshClipperStrategy(kGeom) , m_name(name.empty() ? std::string("FSor") : name) , m_maxRadius(0.0) @@ -32,7 +38,7 @@ MonotonicZSORClipper::MonotonicZSORClipper(const klee::Geometry& kGeom, const st extractClipperInfo(); combineRadialSegments(m_sorCurve); - axom::Array turnIndices = findZSwitchbacks(m_sorCurve.view()); + axom::Array turnIndices = findZSwitchbacks(m_sorCurve.view(), hostAllocator); if(turnIndices.size() > 2) { // The 2 "turns" allowed are the first and last points. Anything else is a switchback. @@ -67,9 +73,25 @@ MonotonicZSORClipper::MonotonicZSORClipper(const klee::Geometry& kGeom, const Point3DType& sorOrigin, const Vector3DType& sorDirection, axom::IndexType levelOfRefinement) + : MonotonicZSORClipper(kGeom, + name, + discreteFunction, + sorOrigin, + sorDirection, + levelOfRefinement, + HostAllocator {}) +{ } + +MonotonicZSORClipper::MonotonicZSORClipper(const klee::Geometry& kGeom, + const std::string& name, + axom::ArrayView discreteFunction, + const Point3DType& sorOrigin, + const Vector3DType& sorDirection, + axom::IndexType levelOfRefinement, + HostAllocator hostAllocator) : MeshClipperStrategy(kGeom) , m_name(name.empty() ? std::string("FSor") : name) - , m_sorCurve(discreteFunction, axom::execution_space::allocatorID()) + , m_sorCurve(discreteFunction, hostAllocator.getID(), hostAllocator) , m_maxRadius(0.0) , m_minRadius(numerics::floating_point_limits::max()) , m_sorOrigin(sorOrigin) @@ -78,7 +100,7 @@ MonotonicZSORClipper::MonotonicZSORClipper(const klee::Geometry& kGeom, , m_transformer() { combineRadialSegments(m_sorCurve); - axom::Array turnIndices = findZSwitchbacks(m_sorCurve.view()); + axom::Array turnIndices = findZSwitchbacks(m_sorCurve.view(), hostAllocator); if(turnIndices.size() > 2) { // The 2 "turns" allowed are the first and last points. Anything else is a switchback. @@ -113,10 +135,15 @@ bool MonotonicZSORClipper::labelCellsInOut(quest::experimental::ShapeMesh& shape SLIC_ERROR_IF(shapeMesh.dimension() != 3, "MonotonicZSORClipper requires a 3D mesh."); const int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); const auto cellCount = shapeMesh.getCellCount(); if(labels.size() < cellCount || labels.getAllocatorID() != allocId) { - labels = axom::Array(ArrayOptions::Uninitialized(), cellCount, cellCount, allocId); + labels = axom::Array(ArrayOptions::Uninitialized(), + cellCount, + cellCount, + allocId, + hostAllocator); } switch(shapeMesh.getRuntimePolicy()) @@ -152,11 +179,13 @@ bool MonotonicZSORClipper::labelTetsInOut(quest::experimental::ShapeMesh& shapeM SLIC_ERROR_IF(shapeMesh.dimension() != 3, "MonotonicZSORClipper requires a 3D mesh."); const int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); const auto cellCount = cellIds.size(); const auto tetCount = cellCount * NUM_TETS_PER_HEX; if(tetLabels.size() < tetCount || tetLabels.getAllocatorID() != allocId) { - tetLabels = axom::Array(ArrayOptions::Uninitialized(), tetCount, tetCount, allocId); + tetLabels = + axom::Array(ArrayOptions::Uninitialized(), tetCount, tetCount, allocId, hostAllocator); } switch(shapeMesh.getRuntimePolicy()) @@ -371,6 +400,7 @@ void MonotonicZSORClipper::computeCurveBoxes(quest::experimental::ShapeMesh& sha * z-axis. */ const int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); const IndexType cellCount = shapeMesh.getCellCount(); axom::ArrayView cellLengths = shapeMesh.getCellLengths(); @@ -391,8 +421,9 @@ void MonotonicZSORClipper::computeCurveBoxes(quest::experimental::ShapeMesh& sha axom::Array sorCurve = subdivideCurve(m_sorCurve, 3 * avgCharLength /* maxMean */, -1 /* maxDz, negative disables */, - -1 /* minDz, negative disables */); - sorCurve = axom::Array(sorCurve, allocId); + -1 /* minDz, negative disables */, + hostAllocator); + sorCurve = axom::Array(sorCurve, allocId, hostAllocator); auto sorCurveView = sorCurve.view(); /* @@ -402,8 +433,8 @@ void MonotonicZSORClipper::computeCurveBoxes(quest::experimental::ShapeMesh& sha Add add to bbOn boxes representing the vertical endcaps of the curve. */ auto segCount = sorCurve.size() - 1; - bbOn = axom::Array(segCount + 2, segCount + 2, allocId); - bbUnder = axom::Array(segCount, segCount, allocId); + bbOn = axom::Array(segCount + 2, segCount + 2, allocId, hostAllocator); + bbUnder = axom::Array(segCount, segCount, allocId, hostAllocator); auto bbOnView = bbOn.view(); auto bbUnderView = bbUnder.view(); @@ -419,7 +450,7 @@ void MonotonicZSORClipper::computeCurveBoxes(quest::experimental::ShapeMesh& sha under = BoundingBox2DType(underMin, underMax); }); - axom::Array endCaps(2, 2); + axom::Array endCaps(2, 2, hostAllocator.getID(), hostAllocator); endCaps[0].addPoint(m_sorCurve.front()); endCaps[0].addPoint(Point2DType {m_sorCurve.front()[0], 0.0}); endCaps[1].addPoint(m_sorCurve.back()); @@ -442,9 +473,10 @@ Array MonotonicZSORClipper::subdivideCurve( const Array& sorCurveIn, double maxMean, double maxDz, - double minDz) + double minDz, + HostAllocator hostAllocator) { - Array sorCurveOut; + Array sorCurveOut(0, 0, hostAllocator.getID(), hostAllocator); if(sorCurveIn.empty()) { @@ -529,7 +561,8 @@ bool MonotonicZSORClipper::getGeometryAsOctsImpl(quest::experimental::ShapeMesh& axom::Array& octs) { const int allocId = shapeMesh.getAllocatorID(); - octs = axom::Array(0, 0, allocId); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); + octs = axom::Array(0, 0, allocId, hostAllocator); const auto cellCount = shapeMesh.getCellCount(); @@ -545,7 +578,8 @@ bool MonotonicZSORClipper::getGeometryAsOctsImpl(quest::experimental::ShapeMesh& axom::Array sorCurve = subdivideCurve(m_sorCurve, 3 * avgCharLength /* maxMean */, 3 * avgCharLength /* maxDz */, - 2 * avgCharLength /* minDz */); + 2 * avgCharLength /* minDz */, + hostAllocator); // Generate the Octahedra int octCount = 0; @@ -553,7 +587,8 @@ bool MonotonicZSORClipper::getGeometryAsOctsImpl(quest::experimental::ShapeMesh& int(sorCurve.size()), m_levelOfRefinement, octs, - octCount); + octCount, + shapeMesh.getHostAllocator()); AXOM_UNUSED_VAR(good); SLIC_ASSERT(good); @@ -645,12 +680,19 @@ void MonotonicZSORClipper::combineRadialSegments(axom::Array& sorCu */ axom::Array MonotonicZSORClipper::findZSwitchbacks( axom::ArrayView pts) +{ + return findZSwitchbacks(pts, HostAllocator {}); +} + +axom::Array MonotonicZSORClipper::findZSwitchbacks( + axom::ArrayView pts, + HostAllocator hostAllocator) { const axom::IndexType segCount = pts.size() - 1; SLIC_ASSERT(segCount > 0); // boundaryIdx is where curve's axial direction changes, plus end points. - axom::Array boundaryIdx(0, 2); + axom::Array boundaryIdx(0, 2, hostAllocator.getID(), hostAllocator); boundaryIdx.push_back(0); constexpr double eps = 1e-14; diff --git a/src/axom/quest/detail/clipping/MonotonicZSORClipper.hpp b/src/axom/quest/detail/clipping/MonotonicZSORClipper.hpp index aba0e98d42..aa0687b718 100644 --- a/src/axom/quest/detail/clipping/MonotonicZSORClipper.hpp +++ b/src/axom/quest/detail/clipping/MonotonicZSORClipper.hpp @@ -6,6 +6,7 @@ #pragma once +#include "axom/core/memory_management.hpp" #include "axom/klee/Geometry.hpp" #include "axom/quest/MeshClipperStrategy.hpp" #include "axom/primal/geometry/CoordinateTransformer.hpp" @@ -49,8 +50,14 @@ class MonotonicZSORClipper : public MeshClipperStrategy * of octs grows exponentially with level * (@see quest::discretize(const axom::ArrayView &polyline, int pointcount, int levels, axom::Array &out, int &octcount)). * In practice, keep this number to 11 or less. + * + * \note The overload without `HostAllocator` is a compatibility path that + * uses Axom's current default host allocator for constructor scratch. */ MonotonicZSORClipper(const klee::Geometry& kGeom, const std::string& name = ""); + MonotonicZSORClipper(const klee::Geometry& kGeom, + const std::string& name, + HostAllocator hostAllocator); /*! * @brief Construct with parameters to override the specifications @@ -63,6 +70,14 @@ class MonotonicZSORClipper : public MeshClipperStrategy const Vector3DType& sorDirection, axom::IndexType levelOfRefinement); + MonotonicZSORClipper(const klee::Geometry& kGeom, + const std::string& name, + axom::ArrayView discreteFunction, + const Point3DType& sorOrigin, + const Vector3DType& sorDirection, + axom::IndexType levelOfRefinement, + HostAllocator hostAllocator); + virtual ~MonotonicZSORClipper() = default; const std::string& name() const override { return m_name; } @@ -91,6 +106,8 @@ class MonotonicZSORClipper : public MeshClipperStrategy * @return Indices of switchbacks, plus the first and last indices. */ static axom::Array findZSwitchbacks(axom::ArrayView pts); + static axom::Array findZSwitchbacks(axom::ArrayView pts, + HostAllocator hostAllocator); /* * @brief Combine consecutive radial segments of the curve into a @@ -207,7 +224,8 @@ class MonotonicZSORClipper : public MeshClipperStrategy axom::Array subdivideCurve(const Array& sorCurveIn, double maxMean, double maxDz, - double minDz); + double minDz, + HostAllocator hostAllocator); //!@brief Compute geometry as octs, by policy. template diff --git a/src/axom/quest/detail/clipping/Plane3DClipper.cpp b/src/axom/quest/detail/clipping/Plane3DClipper.cpp index 6d93ea9cb4..e72aec8edc 100644 --- a/src/axom/quest/detail/clipping/Plane3DClipper.cpp +++ b/src/axom/quest/detail/clipping/Plane3DClipper.cpp @@ -26,10 +26,15 @@ bool Plane3DClipper::labelCellsInOut(quest::experimental::ShapeMesh& shapeMesh, axom::Array& labels) { int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); auto cellCount = shapeMesh.getCellCount(); if(labels.size() < cellCount || labels.getAllocatorID() != shapeMesh.getAllocatorID()) { - labels = axom::Array(ArrayOptions::Uninitialized(), cellCount, cellCount, allocId); + labels = axom::Array(ArrayOptions::Uninitialized(), + cellCount, + cellCount, + allocId, + hostAllocator); } switch(shapeMesh.getRuntimePolicy()) @@ -63,11 +68,13 @@ bool Plane3DClipper::labelTetsInOut(quest::experimental::ShapeMesh& shapeMesh, axom::Array& tetLabels) { int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); const auto cellCount = cellIds.size(); const auto tetCount = cellCount * NUM_TETS_PER_HEX; if(tetLabels.size() < tetCount || tetLabels.getAllocatorID() != allocId) { - tetLabels = axom::Array(ArrayOptions::Uninitialized(), tetCount, tetCount, allocId); + tetLabels = + axom::Array(ArrayOptions::Uninitialized(), tetCount, tetCount, allocId, hostAllocator); } switch(shapeMesh.getRuntimePolicy()) @@ -193,6 +200,7 @@ void Plane3DClipper::labelCellsInOutImpl(quest::experimental::ShapeMesh& shapeMe axom::ArrayView labels) { int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); auto cellCount = shapeMesh.getCellCount(); auto vertCount = shapeMesh.getVertexCount(); auto cellVolumes = shapeMesh.getCellVolumes(); @@ -206,7 +214,11 @@ void Plane3DClipper::labelCellsInOutImpl(quest::experimental::ShapeMesh& shapeMe /* Compute whether vertices are inside shape. */ - axom::Array vertIsInside {ArrayOptions::Uninitialized(), vertCount, vertCount, allocId}; + axom::Array vertIsInside {ArrayOptions::Uninitialized(), + vertCount, + vertCount, + allocId, + hostAllocator}; auto vertIsInsideView = vertIsInside.view(); SLIC_ASSERT(axom::execution_space::usesAllocId(vX.getAllocatorID())); SLIC_ASSERT(axom::execution_space::usesAllocId(vY.getAllocatorID())); @@ -314,7 +326,10 @@ void Plane3DClipper::specializedClipCellsImpl(quest::experimental::ShapeMesh& sh conduit::Node& statistics) { axom::IndexType cellCount = shapeMesh.getCellCount(); - axom::Array cellIds(cellCount, cellCount, shapeMesh.getAllocatorID()); + axom::Array cellIds(cellCount, + cellCount, + shapeMesh.getAllocatorID(), + shapeMesh.getHostAllocator()); auto cellIdsView = cellIds.view(); axom::for_all(cellCount, AXOM_LAMBDA(axom::IndexType i) { cellIdsView[i] = i; }); specializedClipCellsImpl(shapeMesh, ovlap, cellIds, statistics); diff --git a/src/axom/quest/detail/clipping/SORClipper.cpp b/src/axom/quest/detail/clipping/SORClipper.cpp index bafcbe8b66..7a4e2244b2 100644 --- a/src/axom/quest/detail/clipping/SORClipper.cpp +++ b/src/axom/quest/detail/clipping/SORClipper.cpp @@ -22,8 +22,15 @@ namespace experimental { SORClipper::SORClipper(const klee::Geometry& kGeom, const std::string& name) + : SORClipper(kGeom, name, HostAllocator {}) +{ } + +SORClipper::SORClipper(const klee::Geometry& kGeom, const std::string& name, HostAllocator hostAllocator) : MeshClipperStrategy(kGeom) , m_name(name.empty() ? std::string("Sor") : name) + , m_hostAllocator(hostAllocator) + , m_fsorImpls(0, 0, hostAllocator.getID(), hostAllocator) + , m_sorCurve(0, 0, hostAllocator.getID(), hostAllocator) , m_maxRadius(0.0) , m_minRadius(std::numeric_limits::max()) { @@ -44,7 +51,7 @@ SORClipper::SORClipper(const klee::Geometry& kGeom, const std::string& name) MonotonicZSORClipper::combineRadialSegments(m_sorCurve); - axom::Array> sections; + axom::Array> sections(0, 0, hostAllocator.getID(), hostAllocator); splitIntoMonotonicSections(m_sorCurve.view(), sections); for(int i = 0; i < sections.size(); ++i) { @@ -55,7 +62,8 @@ SORClipper::SORClipper(const klee::Geometry& kGeom, const std::string& name) section, m_sorOrigin, m_sorDirection, - m_levelOfRefinement)); + m_levelOfRefinement, + hostAllocator)); } } @@ -76,7 +84,8 @@ bool SORClipper::specializedClipCells(quest::experimental::ShapeMesh& shapeMesh, * correct sign. */ const axom::IndexType cellCount = ovlap.size(); - axom::Array tmpOvlap(cellCount, cellCount, ovlap.getAllocatorID()); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); + axom::Array tmpOvlap(cellCount, cellCount, ovlap.getAllocatorID(), hostAllocator); for(auto& fsorImpl : m_fsorImpls) { tmpOvlap.fill(0.0); @@ -107,7 +116,8 @@ void SORClipper::splitIntoMonotonicSections(axom::ArrayView p axom::Array>& sections) { AXOM_ANNOTATE_SCOPE("SORClipper::splitIntoMonotonicSections"); - axom::Array splitIdx = MonotonicZSORClipper::findZSwitchbacks(pts); + axom::Array splitIdx = + MonotonicZSORClipper::findZSwitchbacks(pts, m_hostAllocator); const axom::IndexType sectionCount = splitIdx.size() - 1; sections.clear(); @@ -117,7 +127,10 @@ void SORClipper::splitIntoMonotonicSections(axom::ArrayView p axom::IndexType firstInSection = splitIdx[i]; axom::IndexType lastInSection = splitIdx[i + 1]; auto& curSection = sections[i]; - curSection.reserve(lastInSection - firstInSection + 1); + curSection = axom::Array(0, + lastInSection - firstInSection + 1, + m_hostAllocator.getID(), + m_hostAllocator); for(axom::IndexType j = firstInSection; j <= lastInSection; ++j) { curSection.push_back(pts[j]); @@ -187,7 +200,11 @@ void SORClipper::extractClipperInfo() "***SORClipper: Discrete function must have an even number of values. It has {}.", n)); - m_sorCurve.resize(axom::ArrayOptions::Uninitialized(), n / 2); + m_sorCurve = axom::Array(axom::ArrayOptions::Uninitialized(), + n / 2, + n / 2, + m_hostAllocator.getID(), + m_hostAllocator); for(int i = 0; i < n / 2; ++i) { m_sorCurve[i] = Point2DType {discreteFunctionArray[i * 2], discreteFunctionArray[i * 2 + 1]}; diff --git a/src/axom/quest/detail/clipping/SORClipper.hpp b/src/axom/quest/detail/clipping/SORClipper.hpp index d0556ef2a8..bf9e462347 100644 --- a/src/axom/quest/detail/clipping/SORClipper.hpp +++ b/src/axom/quest/detail/clipping/SORClipper.hpp @@ -49,6 +49,7 @@ class SORClipper : public MeshClipperStrategy * internal MeshClipper objects. @c MeshClipper::setScreenLevel(). */ SORClipper(const klee::Geometry& kGeom, const std::string& name = ""); + SORClipper(const klee::Geometry& kGeom, const std::string& name, HostAllocator hostAllocator); virtual ~SORClipper() = default; @@ -63,6 +64,8 @@ class SORClipper : public MeshClipperStrategy #endif std::string m_name; + HostAllocator m_hostAllocator; + axom::Array> m_fsorImpls; /*! diff --git a/src/axom/quest/detail/clipping/SphereClipper.cpp b/src/axom/quest/detail/clipping/SphereClipper.cpp index 9c50c6da04..3c865b4e54 100644 --- a/src/axom/quest/detail/clipping/SphereClipper.cpp +++ b/src/axom/quest/detail/clipping/SphereClipper.cpp @@ -32,10 +32,15 @@ bool SphereClipper::labelCellsInOut(quest::experimental::ShapeMesh& shapeMesh, SLIC_ERROR_IF(shapeMesh.dimension() != 3, "SphereClipper requires a 3D mesh."); int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); auto cellCount = shapeMesh.getCellCount(); if(labels.size() < cellCount || labels.getAllocatorID() != shapeMesh.getAllocatorID()) { - labels = axom::Array(ArrayOptions::Uninitialized(), cellCount, cellCount, allocId); + labels = axom::Array(ArrayOptions::Uninitialized(), + cellCount, + cellCount, + allocId, + hostAllocator); } switch(shapeMesh.getRuntimePolicy()) @@ -99,6 +104,7 @@ bool SphereClipper::labelTetsInOut(quest::experimental::ShapeMesh& shapeMesh, const axom::IndexType cellCount = cellIds.size(); const int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); if(tetLabels.size() < cellCount * NUM_TETS_PER_HEX || tetLabels.getAllocatorID() != shapeMesh.getAllocatorID()) @@ -106,7 +112,8 @@ bool SphereClipper::labelTetsInOut(quest::experimental::ShapeMesh& shapeMesh, tetLabels = axom::Array(ArrayOptions::Uninitialized(), cellCount * NUM_TETS_PER_HEX, cellCount * NUM_TETS_PER_HEX, - allocId); + allocId, + hostAllocator); } switch(shapeMesh.getRuntimePolicy()) @@ -231,6 +238,7 @@ bool SphereClipper::getGeometryAsOcts(quest::experimental::ShapeMesh& shapeMesh, auto octsView = octs.view(); auto transformer = m_transformer; int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); axom::for_all( octCount, AXOM_LAMBDA(axom::IndexType iOct) { @@ -245,7 +253,7 @@ bool SphereClipper::getGeometryAsOcts(quest::experimental::ShapeMesh& shapeMesh, // The disretize method uses host data. Place into proper space if needed. if(octs.getAllocatorID() != allocId) { - octs = axom::Array>(octs, allocId); + octs = axom::Array>(octs, allocId, hostAllocator); } SLIC_DEBUG(axom::fmt::format("SphereClipper '{}' {}-level refined got {} geometry octs.", diff --git a/src/axom/quest/detail/clipping/TetClipper.cpp b/src/axom/quest/detail/clipping/TetClipper.cpp index 004c29b7bb..3a72eb96b2 100644 --- a/src/axom/quest/detail/clipping/TetClipper.cpp +++ b/src/axom/quest/detail/clipping/TetClipper.cpp @@ -48,10 +48,15 @@ bool TetClipper::labelCellsInOut(quest::experimental::ShapeMesh& shapeMesh, SLIC_ERROR_IF(shapeMesh.dimension() != 3, "TetClipper requires a 3D mesh."); const int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); const auto cellCount = shapeMesh.getCellCount(); if(cellLabels.size() < cellCount || cellLabels.getAllocatorID() != allocId) { - cellLabels = axom::Array(ArrayOptions::Uninitialized(), cellCount, cellCount, allocId); + cellLabels = axom::Array(ArrayOptions::Uninitialized(), + cellCount, + cellCount, + allocId, + hostAllocator); } switch(shapeMesh.getRuntimePolicy()) @@ -99,6 +104,7 @@ void TetClipper::labelCellsInOutImpl(quest::experimental::ShapeMesh& shapeMesh, */ int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); auto vertCount = shapeMesh.getVertexCount(); auto cellCount = shapeMesh.getCellCount(); auto meshCellVolumes = shapeMesh.getCellVolumes(); @@ -124,8 +130,8 @@ void TetClipper::labelCellsInOutImpl(quest::experimental::ShapeMesh& shapeMesh, axom::ArrayView aboveView[4]; for(IndexType p = 0; p < 4; ++p) { - below[p] = axom::Array(ArrayOptions::Uninitialized(), vertCount, 0, allocId); - above[p] = axom::Array(ArrayOptions::Uninitialized(), vertCount, 0, allocId); + below[p] = axom::Array(ArrayOptions::Uninitialized(), vertCount, 0, allocId, hostAllocator); + above[p] = axom::Array(ArrayOptions::Uninitialized(), vertCount, 0, allocId, hostAllocator); belowView[p] = below[p].view(); aboveView[p] = above[p].view(); } @@ -202,11 +208,13 @@ bool TetClipper::labelTetsInOut(quest::experimental::ShapeMesh& shapeMesh, SLIC_ERROR_IF(shapeMesh.dimension() != 3, "TetClipper requires a 3D mesh."); const int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); const auto cellCount = cellIds.size(); const auto tetCount = cellCount * NUM_TETS_PER_HEX; if(tetLabels.size() < tetCount || tetLabels.getAllocatorID() != allocId) { - tetLabels = axom::Array(ArrayOptions::Uninitialized(), tetCount, tetCount, allocId); + tetLabels = + axom::Array(ArrayOptions::Uninitialized(), tetCount, tetCount, allocId, hostAllocator); } switch(shapeMesh.getRuntimePolicy()) @@ -331,9 +339,10 @@ bool TetClipper::getGeometryAsTets(quest::experimental::ShapeMesh& shapeMesh, { AXOM_ANNOTATE_SCOPE("TetClipper::getGeometryAsTets"); int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); if(tets.getAllocatorID() != allocId || tets.size() != 1) { - tets = axom::Array(1, 1, allocId); + tets = axom::Array(1, 1, allocId, hostAllocator); } // Copy tet into tets array, which may be in non-host memory. axom::copy(tets.data(), &m_tet, sizeof(TetrahedronType)); diff --git a/src/axom/quest/detail/clipping/TetMeshClipper.cpp b/src/axom/quest/detail/clipping/TetMeshClipper.cpp index 1b48d3b25a..826ebeb3d9 100644 --- a/src/axom/quest/detail/clipping/TetMeshClipper.cpp +++ b/src/axom/quest/detail/clipping/TetMeshClipper.cpp @@ -40,10 +40,12 @@ bool TetMeshClipper::labelCellsInOut(quest::experimental::ShapeMesh& shapeMesh, SLIC_ERROR_IF(shapeMesh.dimension() != 3, "TetMeshClipper requires a 3D mesh."); int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); auto cellCount = shapeMesh.getCellCount(); if(labels.size() < cellCount || labels.getAllocatorID() != allocId) { - labels = axom::Array(ArrayOptions::Uninitialized(), cellCount, 0, allocId); + labels = + axom::Array(ArrayOptions::Uninitialized(), cellCount, 0, allocId, hostAllocator); } switch(shapeMesh.getRuntimePolicy()) @@ -79,10 +81,12 @@ bool TetMeshClipper::labelTetsInOut(quest::experimental::ShapeMesh& shapeMesh, SLIC_ERROR_IF(shapeMesh.dimension() != 3, "TetMeshClipper requires a 3D mesh."); int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); const axom::IndexType tetCount = cellIds.size() * NUM_TETS_PER_HEX; if(tetLabels.size() < tetCount || tetLabels.getAllocatorID() != allocId) { - tetLabels = axom::Array(ArrayOptions::Uninitialized(), tetCount, 0, allocId); + tetLabels = + axom::Array(ArrayOptions::Uninitialized(), tetCount, 0, allocId, hostAllocator); } switch(shapeMesh.getRuntimePolicy()) @@ -127,11 +131,12 @@ void TetMeshClipper::labelCellsInOutImpl(quest::experimental::ShapeMesh& shapeMe axom::ArrayView labels) { int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); auto cellCount = shapeMesh.getCellCount(); axom::Array surfTris; spin::BVH<3, ExecSpace, double> bvh; - computeSurfaceTrianglesAndBVH(allocId, surfTris, bvh); + computeSurfaceTrianglesAndBVH(allocId, hostAllocator, surfTris, bvh); auto surfTrisView = surfTris.view(); axom::Array hexRays; @@ -143,17 +148,17 @@ void TetMeshClipper::labelCellsInOutImpl(quest::experimental::ShapeMesh& shapeMe */ axom::ArrayView hexBbs = shapeMesh.getCellBoundingBoxes(); AXOM_ANNOTATE_BEGIN("TetMeshClipper::get_surf_near_bbs"); - axom::Array bbOffsets(cellCount, 0, allocId); - axom::Array bbCounts(cellCount, 0, allocId); + axom::Array bbOffsets(cellCount, 0, allocId, hostAllocator); + axom::Array bbCounts(cellCount, 0, allocId, hostAllocator); axom::Array bbCandidates; - bvh.findBoundingBoxes(bbOffsets, bbCounts, bbCandidates, hexBbs.size(), hexBbs); + bvh.findBoundingBoxes(bbOffsets, bbCounts, bbCandidates, hexBbs.size(), hexBbs, hostAllocator); AXOM_ANNOTATE_END("TetMeshClipper::get_surf_near_bbs"); AXOM_ANNOTATE_BEGIN("TetMeshClipper::get_surf_near_rays"); - axom::Array rayOffsets(cellCount, 0, allocId); - axom::Array rayCounts(cellCount, 0, allocId); + axom::Array rayOffsets(cellCount, 0, allocId, hostAllocator); + axom::Array rayCounts(cellCount, 0, allocId, hostAllocator); axom::Array rayCandidates; - bvh.findRays(rayOffsets, rayCounts, rayCandidates, hexRaysView.size(), hexRaysView); + bvh.findRays(rayOffsets, rayCounts, rayCandidates, hexRaysView.size(), hexRaysView, hostAllocator); AXOM_ANNOTATE_END("TetMeshClipper::get_surf_near_rays"); auto bbCountsView = bbCounts.view(); @@ -252,6 +257,7 @@ void TetMeshClipper::labelTetsInOutImpl(quest::experimental::ShapeMesh& shapeMes axom::ArrayView tetLabels) { int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); auto cellCount = cellIds.size(); auto tetCount = cellCount * NUM_TETS_PER_HEX; @@ -262,7 +268,7 @@ void TetMeshClipper::labelTetsInOutImpl(quest::experimental::ShapeMesh& shapeMes axom::Array surfTris; spin::BVH<3, ExecSpace, double> bvh; - computeSurfaceTrianglesAndBVH(allocId, surfTris, bvh); + computeSurfaceTrianglesAndBVH(allocId, hostAllocator, surfTris, bvh); auto surfTrisView = surfTris.view(); axom::Array tetBbs; @@ -275,17 +281,17 @@ void TetMeshClipper::labelTetsInOutImpl(quest::experimental::ShapeMesh& shapeMes * Find candidate surface triangles near the tets' bounding boxes and rays. */ AXOM_ANNOTATE_BEGIN("TetMeshClipper::get_surf_near_bbs"); - axom::Array bbOffsets(tetCount, 0, allocId); - axom::Array bbCounts(tetCount, 0, allocId); + axom::Array bbOffsets(tetCount, 0, allocId, hostAllocator); + axom::Array bbCounts(tetCount, 0, allocId, hostAllocator); axom::Array bbCandidates; - bvh.findBoundingBoxes(bbOffsets, bbCounts, bbCandidates, tetBbs.size(), tetBbsView); + bvh.findBoundingBoxes(bbOffsets, bbCounts, bbCandidates, tetBbs.size(), tetBbsView, hostAllocator); AXOM_ANNOTATE_END("TetMeshClipper::get_surf_near_bbs"); AXOM_ANNOTATE_BEGIN("TetMeshClipper::get_surf_near_rays"); - axom::Array rayOffsets(tetCount, 0, allocId); - axom::Array rayCounts(tetCount, 0, allocId); + axom::Array rayOffsets(tetCount, 0, allocId, hostAllocator); + axom::Array rayCounts(tetCount, 0, allocId, hostAllocator); axom::Array rayCandidates; - bvh.findRays(rayOffsets, rayCounts, rayCandidates, tetRaysView.size(), tetRaysView); + bvh.findRays(rayOffsets, rayCounts, rayCandidates, tetRaysView.size(), tetRaysView, hostAllocator); AXOM_ANNOTATE_END("TetMeshClipper::get_surf_near_rays"); auto bbCountsView = bbCounts.view(); @@ -390,7 +396,8 @@ void TetMeshClipper::computeHexRays(quest::experimental::ShapeMesh& shapeMesh, Point3DType geomCenter = m_tetMeshBb.getCentroid(); // Estimate of tet mesh center. auto meshHexes = shapeMesh.getCellsAsHexes(); auto cellCount = shapeMesh.getCellCount(); - hexRays = axom::Array(cellCount, 0, shapeMesh.getAllocatorID()); + hexRays = + axom::Array(cellCount, 0, shapeMesh.getAllocatorID(), shapeMesh.getHostAllocator()); auto hexRaysView = hexRays.view(); axom::for_all( cellCount, @@ -428,6 +435,7 @@ void TetMeshClipper::computeTetRays(quest::experimental::ShapeMesh& shapeMesh, { AXOM_ANNOTATE_SCOPE("TetMeshClipper::computeTetRays"); int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); auto cellCount = cellIds.size(); auto tetCount = cellCount * NUM_TETS_PER_HEX; @@ -436,9 +444,14 @@ void TetMeshClipper::computeTetRays(quest::experimental::ShapeMesh& shapeMesh, * away from the center of the tet mesh. */ Point3DType geomCenter = m_tetMeshBb.getCentroid(); // Estimate of tet mesh center. - tetBbs = axom::Array(axom::ArrayOptions::Uninitialized(), tetCount, 0, allocId); + tetBbs = axom::Array(axom::ArrayOptions::Uninitialized(), + tetCount, + 0, + allocId, + hostAllocator); auto tetBbsView = tetBbs.view(); - tetRays = axom::Array(axom::ArrayOptions::Uninitialized(), tetCount, 0, allocId); + tetRays = + axom::Array(axom::ArrayOptions::Uninitialized(), tetCount, 0, allocId, hostAllocator); auto tetRaysView = tetRays.view(); const auto meshTets = shapeMesh.getCellsAsTets(); axom::for_all( @@ -464,6 +477,7 @@ void TetMeshClipper::computeTetRays(quest::experimental::ShapeMesh& shapeMesh, template void TetMeshClipper::computeSurfaceTrianglesAndBVH(int allocId, + HostAllocator hostAllocator, axom::Array& surfTris, spin::BVH<3, ExecSpace, double>& bvh) { @@ -471,7 +485,7 @@ void TetMeshClipper::computeSurfaceTrianglesAndBVH(int allocId, Compute surface triangles of the tet mesh. */ AXOM_ANNOTATE_BEGIN("TetMeshClipper:compute_surface"); - surfTris = computeGeometrySurface(allocId); + surfTris = computeGeometrySurface(allocId, hostAllocator); AXOM_ANNOTATE_END("TetMeshClipper:compute_surface"); auto surfTrisView = surfTris.view(); @@ -479,7 +493,8 @@ void TetMeshClipper::computeSurfaceTrianglesAndBVH(int allocId, Surface triangles (as bounding boxes) in BVH. */ AXOM_ANNOTATE_BEGIN("TetMeshClipper::make_surf_bvh"); - axom::Array surfTrisAsBbs(surfTris.size(), 0, allocId); + bvh.setAllocatorID(allocId); + axom::Array surfTrisAsBbs(surfTris.size(), 0, allocId, hostAllocator); auto surfTrisAsBbsView = surfTrisAsBbs.view(); axom::for_all( @@ -510,10 +525,12 @@ void TetMeshClipper::vertexInsideToCellLabel(quest::experimental::ShapeMesh& sha if(labels.size() < shapeMesh.getCellCount() || labels.getAllocatorID() != shapeMesh.getAllocatorID()) { + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); labels = axom::Array(ArrayOptions::Uninitialized(), shapeMesh.getCellCount(), shapeMesh.getCellCount(), - shapeMesh.getAllocatorID()); + shapeMesh.getAllocatorID(), + hostAllocator); } auto labelsView = labels.view(); @@ -543,9 +560,11 @@ bool TetMeshClipper::getGeometryAsTets(quest::experimental::ShapeMesh& shapeMesh axom::Array& tets) { int allocId = shapeMesh.getAllocatorID(); + HostAllocator hostAllocator = shapeMesh.getHostAllocator(); if(tets.size() < m_tetCount || tets.getAllocatorID() != allocId) { - tets = axom::Array(ArrayOptions::Uninitialized(), m_tetCount, 0, allocId); + tets = + axom::Array(ArrayOptions::Uninitialized(), m_tetCount, 0, allocId, hostAllocator); } switch(shapeMesh.getRuntimePolicy()) @@ -809,7 +828,9 @@ void TetMeshClipper::transformCoordset() * Compute the surface of the tet mesh, using bump utilities. */ template -axom::Array TetMeshClipper::computeGeometrySurface(int allocId) +axom::Array TetMeshClipper::computeGeometrySurface( + int allocId, + HostAllocator hostAllocator) { // Copy some m_tetMesh data to allocId for accessing in ExecSpace. copy_topo_and_coords_to(allocId); @@ -854,8 +875,8 @@ axom::Array TetMeshClipper::computeGeometrySurfa * Compute tet faces as triangles. * Compute rays from triangle centroid, in normal direction. */ - axom::Array faceTris(faceCount, faceCount, allocId); - axom::Array faceRays(faceCount, faceCount, allocId); + axom::Array faceTris(faceCount, faceCount, allocId, hostAllocator); + axom::Array faceRays(faceCount, faceCount, allocId, hostAllocator); auto faceTrisView = faceTris.view(); auto faceRaysView = faceRays.view(); axom::for_all( @@ -875,8 +896,8 @@ axom::Array TetMeshClipper::computeGeometrySurfa /* * Compute whether faces have tets on each side. */ - axom::Array hasCellOnFrontSide(faceCount, 0, allocId); - axom::Array hasCellOnBackSide(faceCount, 0, allocId); + axom::Array hasCellOnFrontSide(faceCount, 0, allocId, hostAllocator); + axom::Array hasCellOnBackSide(faceCount, 0, allocId, hostAllocator); hasCellOnFrontSide.fill(false); hasCellOnBackSide.fill(false); auto hasCellOnFrontSideView = hasCellOnFrontSide.view(); @@ -914,7 +935,11 @@ axom::Array TetMeshClipper::computeGeometrySurfa /* * Mark faces touching only 1 cell. */ - axom::Array hasCellOnOneSide(ArrayOptions::Uninitialized(), faceCount, 0, allocId); + axom::Array hasCellOnOneSide(ArrayOptions::Uninitialized(), + faceCount, + 0, + allocId, + hostAllocator); auto hasCellOnOneSideView = hasCellOnOneSide.view(); axom::for_all( faceCount, @@ -927,7 +952,7 @@ axom::Array TetMeshClipper::computeGeometrySurfa * Get running total of surface triangle count using prefix-sum scan. * Then use the results to populate array of those faces. */ - axom::Array prefixSum(faceCount + 1, 0, allocId); + axom::Array prefixSum(faceCount + 1, 0, allocId, hostAllocator); prefixSum.fill(0); auto prefixSumView = prefixSum.view(); axom::inclusive_scan(hasCellOnOneSide, @@ -936,8 +961,8 @@ axom::Array TetMeshClipper::computeGeometrySurfa axom::IndexType surfFaceCount = -1; axom::copy(&surfFaceCount, prefixSumView.data() + prefixSumView.size() - 1, sizeof(surfFaceCount)); - axom::Array surfFaceIds(surfFaceCount, 0, allocId); - axom::Array surfTris(surfFaceCount, 0, allocId); + axom::Array surfFaceIds(surfFaceCount, 0, allocId, hostAllocator); + axom::Array surfTris(surfFaceCount, 0, allocId, hostAllocator); auto surfFaceIdsView = surfFaceIds.view(); auto surfTrisView = surfTris.view(); axom::for_all( @@ -964,7 +989,8 @@ void TetMeshClipper::writeTrianglesToVTK(const axom::Array& tria return; } - axom::Array hostTriangles(triangles, axom::MALLOC_ALLOCATOR_ID); + HostAllocator mallocHostAllocator(axom::MALLOC_ALLOCATOR_ID); + axom::Array hostTriangles(triangles, axom::MALLOC_ALLOCATOR_ID, mallocHostAllocator); // Header ofs << "# vtk DataFile Version 3.0\n"; diff --git a/src/axom/quest/detail/clipping/TetMeshClipper.hpp b/src/axom/quest/detail/clipping/TetMeshClipper.hpp index 2c83238029..28b8d85de7 100644 --- a/src/axom/quest/detail/clipping/TetMeshClipper.hpp +++ b/src/axom/quest/detail/clipping/TetMeshClipper.hpp @@ -145,6 +145,7 @@ class TetMeshClipper : public MeshClipperStrategy */ template void computeSurfaceTrianglesAndBVH(int allocId, + HostAllocator hostAllocator, axom::Array& surfTris, spin::BVH<3, ExecSpace, double>& bvh); @@ -152,7 +153,7 @@ class TetMeshClipper : public MeshClipperStrategy * @brief Compute the tet-mesh geometry surface as trianglular facets. */ template - axom::Array computeGeometrySurface(int allocId); + axom::Array computeGeometrySurface(int allocId, HostAllocator hostAllocator); /*! * @brief Add a polyhedral topology to an unstructured tet mesh. diff --git a/src/axom/quest/detail/shaping/PrimitiveSampler.hpp b/src/axom/quest/detail/shaping/PrimitiveSampler.hpp index 1e220fc230..bcaf78791a 100644 --- a/src/axom/quest/detail/shaping/PrimitiveSampler.hpp +++ b/src/axom/quest/detail/shaping/PrimitiveSampler.hpp @@ -49,7 +49,8 @@ class PrimitiveSampler public: /** - * \brief Constructor for a PrimitiveSampler over a collection of triangles (in 2D) or tetrahedra (in 3D) + * \brief Constructor for a PrimitiveSampler over a collection of triangles + * (in 2D) or tetrahedra (in 3D) * * \param shapeName The name of the shape; will be used for the field for the associated samples * \param surfaceMesh Pointer to the surface mesh @@ -57,7 +58,24 @@ class PrimitiveSampler * \note Does not take ownership of the surface mesh */ PrimitiveSampler(const std::string& shapeName, std::shared_ptr surfaceMesh) + : PrimitiveSampler(shapeName, surfaceMesh, HostAllocator {}) + { } + + /** + * \brief Constructor for a PrimitiveSampler over a collection of triangles + * (in 2D) or tetrahedra (in 3D) + * + * \param shapeName The name of the shape; will be used for the field for the associated samples + * \param surfaceMesh Pointer to the surface mesh + * \param hostAllocator Allocator to use for host-accessible scratch and staging + * + * \note Does not take ownership of the surface mesh + */ + PrimitiveSampler(const std::string& shapeName, + std::shared_ptr surfaceMesh, + HostAllocator hostAllocator) : m_shapeName(shapeName) + , m_hostAllocator(hostAllocator) , m_surfaceMesh(surfaceMesh) { } @@ -100,12 +118,18 @@ class PrimitiveSampler // extract the primitives and their bounding boxes const int num_cells = pmesh->getNumberOfCells(); - m_aabbs.resize(num_cells); - m_primitives.resize(num_cells); + axom::Array aabbs_host(num_cells, + num_cells, + m_hostAllocator.getID(), + m_hostAllocator); + axom::Array primitives_host(num_cells, + num_cells, + m_hostAllocator.getID(), + m_hostAllocator); for(int i = 0; i < num_cells; ++i) { const axom::IndexType* connec = pmesh->getCellNodeIDs(i); - auto& simplex = m_primitives[i]; + auto& simplex = primitives_host[i]; for(int j = 0; j < NDIMS + 1; ++j) { simplex[j] = verts[connec[j]]; @@ -128,10 +152,14 @@ class PrimitiveSampler // TODO: WE should only consider simplices in the bounding box of the current domain if(!is_degenerate) { - m_aabbs[i] = primal::compute_bounding_box(simplex); + aabbs_host[i] = primal::compute_bounding_box(simplex); } } + const int allocatorID = axom::execution_space::allocatorID(); + m_aabbs = axom::Array(aabbs_host, allocatorID, m_hostAllocator); + m_primitives = axom::Array(primitives_host, allocatorID, m_hostAllocator); + SLIC_INFO_ROOT("Mesh bounding box: " << m_bbox); // Print out the total volume of all the tetrahedra @@ -223,7 +251,8 @@ class PrimitiveSampler // Get the positions of the query points, project them if needed axom::ArrayView orig_qpts_v(reinterpret_cast(pos_coef->HostReadWrite()), nq); - axom::Array projected_qpts(0); + const int allocatorID = axom::execution_space::allocatorID(); + axom::Array projected_qpts(0, 0, allocatorID, m_hostAllocator); if(projector) { AXOM_ANNOTATE_SCOPE("project query points"); @@ -243,11 +272,11 @@ class PrimitiveSampler axom::ArrayView inout_view(const_cast(inout->HostRead()), nq); axom::for_all(nq, AXOM_LAMBDA(axom::IndexType i) { inout_view[i] = 0.; }); - axom::Array offsets(nq, nq); - axom::Array counts(nq, nq); + axom::Array offsets(nq, nq, allocatorID, m_hostAllocator); + axom::Array counts(nq, nq, allocatorID, m_hostAllocator); axom::Array candidates; - m_bvh.findPoints(offsets.view(), counts.view(), candidates, nq, query_view); + m_bvh.findPoints(offsets.view(), counts.view(), candidates, nq, query_view, m_hostAllocator); auto counts_view = counts.view(); auto offsets_view = offsets.view(); @@ -320,6 +349,7 @@ class PrimitiveSampler DISABLE_MOVE_AND_ASSIGNMENT(PrimitiveSampler); std::string m_shapeName; + HostAllocator m_hostAllocator; BVHType m_bvh; diff --git a/src/axom/quest/detail/shaping/WindingNumberSampler.hpp b/src/axom/quest/detail/shaping/WindingNumberSampler.hpp index 23aff52ccd..03465768d5 100644 --- a/src/axom/quest/detail/shaping/WindingNumberSampler.hpp +++ b/src/axom/quest/detail/shaping/WindingNumberSampler.hpp @@ -84,7 +84,22 @@ class WindingNumberSampler * \param geomView A view that contains the shapes being queried. * */ - WindingNumberSampler(const std::string& shapeName, GeometryView geomView) : m_shapeName(shapeName) + WindingNumberSampler(const std::string& shapeName, GeometryView geomView) + : WindingNumberSampler(shapeName, geomView, HostAllocator {}) + { } + + /*! + * \brief Constructor for a WindingNumberSampler + * + * \param shapeName The name of the shape; will be used for the field for the associated samples + * \param geomView A view that contains the shapes being queried. + * \param hostAllocator Allocator to use for host-accessible scratch and staging + * + */ + WindingNumberSampler(const std::string& shapeName, GeometryView geomView, HostAllocator hostAllocator) + : m_shapeName(shapeName) + , m_hostAllocator(hostAllocator) + , m_contourCaches(0, 0, m_hostAllocator.getID(), m_hostAllocator) { for(const auto& contour : geomView) { @@ -109,9 +124,8 @@ class WindingNumberSampler // Figure out bounding boxes for each geometric object. const axom::IndexType geometrySize = m_contourCaches.size(); - axom::Array aabbs(geometrySize, - geometrySize, - axom::execution_space::allocatorID()); + const auto allocatorID = axom::execution_space::allocatorID(); + axom::Array aabbs(geometrySize, geometrySize, allocatorID, m_hostAllocator); auto aabbsView = aabbs.view(); const auto contourCaches = m_contourCaches; axom::for_all( @@ -194,7 +208,7 @@ class WindingNumberSampler axom::utilities::Timer timer(true); AXOM_ANNOTATE_BEGIN("Create query points"); const auto allocatorID = axom::execution_space::allocatorID(); - axom::Array queryPoints(numQueryPoints, numQueryPoints, allocatorID); + axom::Array queryPoints(numQueryPoints, numQueryPoints, allocatorID, m_hostAllocator); auto queryPointsView = queryPoints.view(); axom::for_all( numQueryPoints, @@ -209,17 +223,17 @@ class WindingNumberSampler // Look up all of the query points. This will allocate the candidates array. AXOM_ANNOTATE_BEGIN("findPoints"); - axom::Array offsets(numQueryPoints, numQueryPoints, allocatorID); - axom::Array sizes(numQueryPoints, numQueryPoints, allocatorID); + axom::Array offsets(numQueryPoints, numQueryPoints, allocatorID, m_hostAllocator); + axom::Array sizes(numQueryPoints, numQueryPoints, allocatorID, m_hostAllocator); axom::Array candidates; auto offsetsView = offsets.view(); auto sizesView = sizes.view(); - m_bvh.findPoints(offsetsView, sizesView, candidates, numQueryPoints, queryPointsView); + m_bvh.findPoints(offsetsView, sizesView, candidates, numQueryPoints, queryPointsView, m_hostAllocator); AXOM_ANNOTATE_END("findPoints"); // Check each element's quad points for in/out. AXOM_ANNOTATE_BEGIN("InOut tests"); - axom::Array inOutResult(numQueryPoints, numQueryPoints, allocatorID); + axom::Array inOutResult(numQueryPoints, numQueryPoints, allocatorID, m_hostAllocator); auto inOutResultView = inOutResult.view(); const auto candidatesView = candidates.view(); const auto contourCaches = m_contourCaches; @@ -324,6 +338,7 @@ class WindingNumberSampler DISABLE_MOVE_AND_ASSIGNMENT(WindingNumberSampler); std::string m_shapeName; + HostAllocator m_hostAllocator; GeometricBoundingBox m_bbox {}; ContourCacheArray m_contourCaches; BVH m_bvh {}; diff --git a/src/axom/quest/docs/sphinx/index.rst b/src/axom/quest/docs/sphinx/index.rst index 01e2606598..29b4a612e9 100644 --- a/src/axom/quest/docs/sphinx/index.rst +++ b/src/axom/quest/docs/sphinx/index.rst @@ -38,6 +38,16 @@ on a ``mint::Mesh``. - :ref:`Isosurface detection`: generate an isosurface mesh from a nodal scalar field and an isovalue. +Host Allocator Selection +------------------------ + +Quest workflows that allocate host-resident scratch or stage data for device +execution provide explicit ``axom::HostAllocator`` paths. Prefer these +overloads in new code when host allocator ownership is available, such as when +constructing shaping, clipping, point-finding, or candidate-query helpers. +Existing overloads without a host allocator remain compatibility paths and use +Axom's current default host allocator. + API Documentation ----------------- @@ -56,4 +66,3 @@ Doxygen generated API documentation can be found here: `API documentation <../.. point_in_cell all_nearest_neighbors isosurface_detection - diff --git a/src/axom/quest/examples/point_in_cell_benchmark.cpp b/src/axom/quest/examples/point_in_cell_benchmark.cpp index 84e1690ea4..04bf70b2ff 100644 --- a/src/axom/quest/examples/point_in_cell_benchmark.cpp +++ b/src/axom/quest/examples/point_in_cell_benchmark.cpp @@ -187,6 +187,7 @@ void benchmark_point_in_cell(mfem::Mesh& mesh, const Arguments& args) // Get ids of necessary allocators constexpr bool on_device = axom::execution_space::onDevice(); const int host_allocator = axom::execution_space::allocatorID(); + const axom::HostAllocator hostAllocator {host_allocator}; const int device_allocator = axom::execution_space::allocatorID(); BoxType meshBb; @@ -198,7 +199,7 @@ void benchmark_point_in_cell(mfem::Mesh& mesh, const Arguments& args) } SLIC_DEBUG("Mesh bounding box " << meshBb); - axom::Array pts_h(npts, npts, host_allocator); + axom::Array pts_h(npts, npts, host_allocator, hostAllocator); // Generate random points utilities::Timer timer(true); @@ -215,7 +216,11 @@ void benchmark_point_in_cell(mfem::Mesh& mesh, const Arguments& args) // Initialize the spatial index timer.start(); - quest::PointInCell query(&mesh, bins.data()); + quest::PointInCell query(&mesh, + bins.data(), + 1e-8, + device_allocator, + hostAllocator); query.setPrintLevel(args.verbosity); query.setInitialGuessType(args.init_guess_type); query.setInitialGridOrder(args.init_guess_order); @@ -224,13 +229,13 @@ void benchmark_point_in_cell(mfem::Mesh& mesh, const Arguments& args) SLIC_INFO(axom::fmt::format("Initialized point-in-cell query in {} s.", timer.elapsed())); // Run query - axom::Array outCellIds_d(npts, npts, device_allocator); - axom::Array outIsoParams_d(npts, npts, device_allocator); + axom::Array outCellIds_d(npts, npts, device_allocator, hostAllocator); + axom::Array outIsoParams_d(npts, npts, device_allocator, hostAllocator); auto outCellIds_v = outCellIds_d.view(); auto outIsoParams_v = outIsoParams_d.view(); - axom::Array pts_d = axom::Array(pts_h, device_allocator); + axom::Array pts_d = axom::Array(pts_h, device_allocator, hostAllocator); timer.start(); query.locatePoints(pts_d.view(), outCellIds_v.data(), outIsoParams_v.data()); @@ -242,10 +247,12 @@ void benchmark_point_in_cell(mfem::Mesh& mesh, const Arguments& args) npts / time)); // Copy back to host - axom::Array outCellIds_h = - on_device ? axom::Array(outCellIds_d, host_allocator) : std::move(outCellIds_d); - axom::Array outIsoParams_h = - on_device ? axom::Array(outIsoParams_d, host_allocator) : std::move(outIsoParams_d); + axom::Array outCellIds_h = on_device + ? axom::Array(outCellIds_d, host_allocator, hostAllocator) + : std::move(outCellIds_d); + axom::Array outIsoParams_h = on_device + ? axom::Array(outIsoParams_d, host_allocator, hostAllocator) + : std::move(outIsoParams_d); // Verify the results by reconstructing physical points from refrerence coordinates if(verifyPoints) diff --git a/src/axom/quest/examples/quest_candidates_example.cpp b/src/axom/quest/examples/quest_candidates_example.cpp index d8876cf974..f83df56727 100644 --- a/src/axom/quest/examples/quest_candidates_example.cpp +++ b/src/axom/quest/examples/quest_candidates_example.cpp @@ -263,6 +263,7 @@ HexMesh loadBlueprintHexMesh(const std::string& mesh_path, // Get ids of necessary allocators const int host_allocator = axom::getUmpireResourceAllocatorID(umpire::resource::Host); + const axom::HostAllocator hostAllocator {host_allocator}; const int kernel_allocator = on_device ? axom::getUmpireResourceAllocatorID(umpire::resource::Device) : axom::execution_space::allocatorID(); @@ -338,16 +339,19 @@ HexMesh loadBlueprintHexMesh(const std::string& mesh_path, auto z_vals_h = axom::ArrayView(n_load[0]["coordsets/coords/values/z"].value(), num_nodes); // Move xyz values onto device - axom::Array x_vals_d = - on_device ? axom::Array(x_vals_h, kernel_allocator) : axom::Array(); + axom::Array x_vals_d = on_device + ? axom::Array(x_vals_h, kernel_allocator, hostAllocator) + : axom::Array(); auto x_vals_view = on_device ? x_vals_d.view() : x_vals_h; - axom::Array y_vals_d = - on_device ? axom::Array(y_vals_h, kernel_allocator) : axom::Array(); + axom::Array y_vals_d = on_device + ? axom::Array(y_vals_h, kernel_allocator, hostAllocator) + : axom::Array(); auto y_vals_view = on_device ? y_vals_d.view() : y_vals_h; - axom::Array z_vals_d = - on_device ? axom::Array(z_vals_h, kernel_allocator) : axom::Array(); + axom::Array z_vals_d = on_device + ? axom::Array(z_vals_h, kernel_allocator, hostAllocator) + : axom::Array(); auto z_vals_view = on_device ? z_vals_d.view() : z_vals_h; // Move connectivity information onto device @@ -373,17 +377,18 @@ HexMesh loadBlueprintHexMesh(const std::string& mesh_path, auto connectivity_h = axom::ArrayView(conn_data, connectivity_size); - axom::Array connectivity_d = - on_device ? axom::Array(connectivity_h, kernel_allocator) : axom::Array(); + axom::Array connectivity_d = on_device + ? axom::Array(connectivity_h, kernel_allocator, hostAllocator) + : axom::Array(); auto connectivity_view = on_device ? connectivity_d.view() : connectivity_h; // Initialize hex elements and bounding boxes const int numCells = connectivity_size / HEX_OFFSET; - hexMesh.m_hexes = HexArray(numCells, numCells, kernel_allocator); + hexMesh.m_hexes = HexArray(numCells, numCells, kernel_allocator, hostAllocator); auto m_hexes_v = (hexMesh.m_hexes).view(); - hexMesh.m_hexBoundingBoxes = BBoxArray(numCells, numCells, kernel_allocator); + hexMesh.m_hexBoundingBoxes = BBoxArray(numCells, numCells, kernel_allocator, hostAllocator); auto m_hexBoundingBoxes_v = (hexMesh.m_hexBoundingBoxes).view(); axom::for_all( @@ -411,8 +416,9 @@ HexMesh loadBlueprintHexMesh(const std::string& mesh_path, }); // Initialize mesh's bounding box on the host - BBoxArray hexBoundingBoxes_h = - on_device ? BBoxArray(hexMesh.m_hexBoundingBoxes, host_allocator) : hexMesh.m_hexBoundingBoxes; + BBoxArray hexBoundingBoxes_h = on_device + ? BBoxArray(hexMesh.m_hexBoundingBoxes, host_allocator, hostAllocator) + : hexMesh.m_hexBoundingBoxes; for(const auto& hexbb : hexBoundingBoxes_h) { hexMesh.m_meshBoundingBox.addBox(hexbb); @@ -476,6 +482,7 @@ std::vector findCandidatesBVH(const HexMesh& insertMesh, const HexMes // Get ids of necessary allocators const int host_allocator = axom::getUmpireResourceAllocatorID(umpire::resource::Host); + const axom::HostAllocator hostAllocator {host_allocator}; const int kernel_allocator = on_device ? axom::getUmpireResourceAllocatorID(umpire::resource::Device) : axom::execution_space::allocatorID(); @@ -493,13 +500,18 @@ std::vector findCandidatesBVH(const HexMesh& insertMesh, const HexMes // Search for candidate bounding boxes of hexes to query; AXOM_ANNOTATE_BEGIN("query candidates"); - IndexArray offsets_d(query_bbox_v.size(), query_bbox_v.size(), kernel_allocator); - IndexArray counts_d(query_bbox_v.size(), query_bbox_v.size(), kernel_allocator); - IndexArray candidates_d(0, 0, kernel_allocator); + IndexArray offsets_d(query_bbox_v.size(), query_bbox_v.size(), kernel_allocator, hostAllocator); + IndexArray counts_d(query_bbox_v.size(), query_bbox_v.size(), kernel_allocator, hostAllocator); + IndexArray candidates_d(0, 0, kernel_allocator, hostAllocator); auto offsets_v = offsets_d.view(); auto counts_v = counts_d.view(); - bvh.findBoundingBoxes(offsets_v, counts_v, candidates_d, query_bbox_v.size(), query_bbox_v); + bvh.findBoundingBoxes(offsets_v, + counts_v, + candidates_d, + query_bbox_v.size(), + query_bbox_v, + hostAllocator); SLIC_INFO(axom::fmt::format("1: Queried candidate bounding boxes.")); AXOM_ANNOTATE_END("query candidates"); @@ -508,8 +520,8 @@ std::vector findCandidatesBVH(const HexMesh& insertMesh, const HexMes // Initialize candidate pairs on device. auto candidates_v = candidates_d.view(); - IndexArray firstPair_d(candidates_v.size(), candidates_v.size(), kernel_allocator); - IndexArray secondPair_d(candidates_v.size(), candidates_v.size(), kernel_allocator); + IndexArray firstPair_d(candidates_v.size(), candidates_v.size(), kernel_allocator, hostAllocator); + IndexArray secondPair_d(candidates_v.size(), candidates_v.size(), kernel_allocator, hostAllocator); auto first_pair_v = firstPair_d.view(); auto second_pair_v = secondPair_d.view(); @@ -532,8 +544,9 @@ std::vector findCandidatesBVH(const HexMesh& insertMesh, const HexMes // copy pairs back to host and into return array AXOM_ANNOTATE_BEGIN("copy pairs to host"); - IndexArray candidates_h[2] = {on_device ? IndexArray(firstPair_d, host_allocator) : IndexArray(), - on_device ? IndexArray(secondPair_d, host_allocator) : IndexArray()}; + IndexArray candidates_h[2] = { + on_device ? IndexArray(firstPair_d, host_allocator, hostAllocator) : IndexArray(), + on_device ? IndexArray(secondPair_d, host_allocator, hostAllocator) : IndexArray()}; auto candidate1_h_v = on_device ? candidates_h[0].view() : firstPair_d.view(); auto candidate2_h_v = on_device ? candidates_h[1].view() : secondPair_d.view(); @@ -579,6 +592,7 @@ std::vector findCandidatesImplicit(const HexMesh& insertMesh, // Get ids of necessary allocators const int host_allocator = axom::getUmpireResourceAllocatorID(umpire::resource::Host); + const axom::HostAllocator hostAllocator {host_allocator}; const int kernel_allocator = on_device ? axom::getUmpireResourceAllocatorID(umpire::resource::Device) : axom::execution_space::allocatorID(); @@ -608,8 +622,8 @@ std::vector findCandidatesImplicit(const HexMesh& insertMesh, AXOM_ANNOTATE_END("initializing implicit grid"); AXOM_ANNOTATE_BEGIN("query candidates"); - IndexArray offsets_d(query_bbox_v.size(), query_bbox_v.size(), kernel_allocator); - IndexArray counts_d(query_bbox_v.size(), query_bbox_v.size(), kernel_allocator); + IndexArray offsets_d(query_bbox_v.size(), query_bbox_v.size(), kernel_allocator, hostAllocator); + IndexArray counts_d(query_bbox_v.size(), query_bbox_v.size(), kernel_allocator, hostAllocator); auto offsets_v = offsets_d.view(); auto counts_v = counts_d.view(); @@ -657,8 +671,14 @@ std::vector findCandidatesImplicit(const HexMesh& insertMesh, // Initialize candidatePairs to return. // Allocate arrays for candidate pairs - IndexArray firstPair_d(totalCandidatePairs.get(), totalCandidatePairs.get(), kernel_allocator); - IndexArray secondPair_d(totalCandidatePairs.get(), totalCandidatePairs.get(), kernel_allocator); + IndexArray firstPair_d(totalCandidatePairs.get(), + totalCandidatePairs.get(), + kernel_allocator, + hostAllocator); + IndexArray secondPair_d(totalCandidatePairs.get(), + totalCandidatePairs.get(), + kernel_allocator, + hostAllocator); auto first_pair_v = firstPair_d.view(); auto second_pair_v = secondPair_d.view(); @@ -687,8 +707,9 @@ std::vector findCandidatesImplicit(const HexMesh& insertMesh, // copy results back to host and into return vector AXOM_ANNOTATE_BEGIN("copy pairs to host"); - IndexArray candidates_h[2] = {on_device ? IndexArray(firstPair_d, host_allocator) : IndexArray(), - on_device ? IndexArray(secondPair_d, host_allocator) : IndexArray()}; + IndexArray candidates_h[2] = { + on_device ? IndexArray(firstPair_d, host_allocator, hostAllocator) : IndexArray(), + on_device ? IndexArray(secondPair_d, host_allocator, hostAllocator) : IndexArray()}; auto candidate1_h_v = on_device ? candidates_h[0].view() : firstPair_d.view(); auto candidate2_h_v = on_device ? candidates_h[1].view() : secondPair_d.view(); diff --git a/src/axom/quest/examples/quest_distributed_distance_query_example.cpp b/src/axom/quest/examples/quest_distributed_distance_query_example.cpp index bb39bb93cf..c7e34c2182 100644 --- a/src/axom/quest/examples/quest_distributed_distance_query_example.cpp +++ b/src/axom/quest/examples/quest_distributed_distance_query_example.cpp @@ -1276,6 +1276,7 @@ int main(int argc, char** argv) auto& rm = umpire::ResourceManager::getInstance(); umpire::Allocator umpireAllocator = rm.getAllocator(umpireResourceName); #endif + const axom::HostAllocator hostAllocator {axom::execution_space::allocatorID()}; // Storage for meshes. sidre::DataStore dataStore; @@ -1377,6 +1378,7 @@ int main(int argc, char** argv) #if defined(AXOM_USE_UMPIRE) query.setAllocatorID(umpireAllocator.getId()); #endif + query.setHostAllocator(hostAllocator); query.setMpiCommunicator(MPI_COMM_WORLD, true); query.setVerbosity(params.isVerbose()); query.setDistanceThreshold(params.distThreshold); diff --git a/src/axom/quest/examples/quest_marching_cubes_example.cpp b/src/axom/quest/examples/quest_marching_cubes_example.cpp index 3c19f9734f..881842d8d8 100644 --- a/src/axom/quest/examples/quest_marching_cubes_example.cpp +++ b/src/axom/quest/examples/quest_marching_cubes_example.cpp @@ -692,6 +692,11 @@ static void addToStackArray(axom::StackArray& a, U b) } } +axom::HostAllocator hostAllocatorToTest() +{ + return axom::HostAllocator {axom::execution_space::allocatorID()}; +} + /*! @brief Strategy pattern for supporting a variety of contour types. @@ -841,8 +846,11 @@ struct ContourTestBase { AXOM_ANNOTATE_SCOPE("MCInit"); initializationTimer.start(); - mcPtr = - std::make_unique(params.policy, s_allocatorId, params.dataParallelism); + const axom::HostAllocator hostAllocator = hostAllocatorToTest(); + mcPtr = std::make_unique(params.policy, + s_allocatorId, + hostAllocator, + params.dataParallelism); mcPtr->setMesh(computationalMesh.asConduitNode(), "mesh", "mask"); initializationTimer.stop(); } @@ -1611,10 +1619,10 @@ int allocatorIdToTest(axom::runtime_policy::Policy policy) //--------------------------------------------------------------------------- // Memory resource. For testing, choose device memory if appropriate. //--------------------------------------------------------------------------- - int allocatorID = - policy == RuntimePolicy::seq ? axom::detail::getAllocatorID() : + const axom::HostAllocator hostAllocator {axom::execution_space::allocatorID()}; + int allocatorID = policy == RuntimePolicy::seq ? hostAllocator.getID() : #if defined(AXOM_RUNTIME_POLICY_USE_OPENMP) - policy == RuntimePolicy::omp ? axom::detail::getAllocatorID() + policy == RuntimePolicy::omp ? hostAllocator.getID() : #endif #if defined(AXOM_RUNTIME_POLICY_USE_CUDA) diff --git a/src/axom/quest/examples/quest_winding_number_2d.cpp b/src/axom/quest/examples/quest_winding_number_2d.cpp index d3f5380d05..1989af4d74 100644 --- a/src/axom/quest/examples/quest_winding_number_2d.cpp +++ b/src/axom/quest/examples/quest_winding_number_2d.cpp @@ -464,6 +464,8 @@ GWNQueryType make_gwn_query(axom::runtime_policy::Policy policy, SLIC_INFO(axom::fmt::format("Using policy omp with {} threads", omp_get_max_threads())); return pick_gwn_method(linearize_curves, approximation_order); } +#else + AXOM_UNUSED_VAR(policy); #endif SLIC_INFO("Using policy seq"); diff --git a/src/axom/quest/examples/quest_winding_number_3d.cpp b/src/axom/quest/examples/quest_winding_number_3d.cpp index 16f14be0a6..8849c7177c 100644 --- a/src/axom/quest/examples/quest_winding_number_3d.cpp +++ b/src/axom/quest/examples/quest_winding_number_3d.cpp @@ -298,6 +298,8 @@ GWNQueryType make_gwn_query(axom::runtime_policy::Policy policy, SLIC_INFO(axom::fmt::format("Using policy omp with {} threads", omp_get_max_threads())); return pick_gwn_method(triangulate, approximation_order); } +#else + AXOM_UNUSED_VAR(policy); #endif SLIC_INFO("Using policy seq"); diff --git a/src/axom/quest/tests/quest_discretize.cpp b/src/axom/quest/tests/quest_discretize.cpp index 4c49fe3314..8195a40758 100644 --- a/src/axom/quest/tests/quest_discretize.cpp +++ b/src/axom/quest/tests/quest_discretize.cpp @@ -763,6 +763,34 @@ TEST(quest_discretize, segment_test) #endif } +//------------------------------------------------------------------------------ +TEST(quest_discretize, segment_test_explicit_host_allocator) +{ + constexpr int generations = 2; + constexpr int hostAllocID = axom::MALLOC_ALLOCATOR_ID; + + axom::Array polyline(2, 2, hostAllocID); + polyline[0] = Point2D {1.0, 1.0}; + polyline[1] = Point2D {1.8, 0.8}; + + axom::Array generated; + int octcount = 0; + axom::quest::discretize(polyline, + 2, + generations, + generated, + octcount, + axom::HostAllocator {hostAllocID}); + + axom::Array handcut; + discretized_segment(polyline[0], polyline[1], handcut); + + EXPECT_EQ(octcount, handcut.size()); + EXPECT_TRUE(check_generation(handcut, generated, 0, 0, 1)); + EXPECT_TRUE(check_generation(handcut, generated, 1, 1, 3)); + EXPECT_TRUE(check_generation(handcut, generated, 2, 4, 6)); +} + //------------------------------------------------------------------------------ TEST(quest_discretize, multi_segment_test) { diff --git a/src/axom/quest/tests/quest_intersection_shaper.cpp b/src/axom/quest/tests/quest_intersection_shaper.cpp index 66208623ec..6245f57d44 100644 --- a/src/axom/quest/tests/quest_intersection_shaper.cpp +++ b/src/axom/quest/tests/quest_intersection_shaper.cpp @@ -130,6 +130,19 @@ TEST(IntersectionShaperTest, case1_seq) replacementRuleTestSet(case1, "seq", RuntimePolicy::seq, tolerance); } } + +TEST(IntersectionShaperTest, case1_seq_explicit_host_allocator) +{ + if(testApp.selected("seq", 1)) + { + replacementRuleTest(case1.front(), + "seq", + RuntimePolicy::seq, + tolerance, + false, + axom::HostAllocator {axom::MALLOC_ALLOCATOR_ID}); + } +} #endif #if defined(RUN_AXOM_OMP_TESTS) TEST(IntersectionShaperTest, case1_omp) diff --git a/src/axom/quest/tests/quest_intersection_shaper_utils.hpp b/src/axom/quest/tests/quest_intersection_shaper_utils.hpp index 853f7b525b..b4c5506496 100644 --- a/src/axom/quest/tests/quest_intersection_shaper_utils.hpp +++ b/src/axom/quest/tests/quest_intersection_shaper_utils.hpp @@ -243,7 +243,8 @@ void replacementRuleTest(const std::string &shapeFile, const std::string &policyName, RuntimePolicy policy, double tolerance, - bool initialMats = false) + bool initialMats = false, + axom::HostAllocator hostAllocator = axom::HostAllocator {}) { // Make potential baseline filenames for this test. Make a policy-specific // baseline that we can check first. If it is not present, the next baseline is tried. @@ -276,7 +277,7 @@ void replacementRuleTest(const std::string &shapeFile, // data collection communicator gets set to MPI_COMM_NULL, which is bad for the C2C reader. dc.SetComm(MPI_COMM_WORLD); #endif - quest::IntersectionShaper shaper(policy, axom::INVALID_ALLOCATOR_ID, shapeSet, &dc); + quest::IntersectionShaper shaper(policy, axom::INVALID_ALLOCATOR_ID, hostAllocator, shapeSet, &dc); shaper.setLevel(refinementLevel); // Borrowed from shaping_driver. diff --git a/src/axom/quest/tests/quest_point_in_cell_mfem.cpp b/src/axom/quest/tests/quest_point_in_cell_mfem.cpp index 780f085d49..3824127ce2 100644 --- a/src/axom/quest/tests/quest_point_in_cell_mfem.cpp +++ b/src/axom/quest/tests/quest_point_in_cell_mfem.cpp @@ -38,6 +38,7 @@ #include #include #include +#include namespace { @@ -58,6 +59,83 @@ const int NUM_TEST_PTS = 10000; const int TEST_GRID_RES = 3; #endif +#if defined(AXOM_USE_UMPIRE) +bool runtimeMemorySpaceAvailable(axom::MemorySpace space) +{ + try + { + switch(space) + { + case axom::MemorySpace::Host: + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); + break; + case axom::MemorySpace::Device: + #if defined(UMPIRE_ENABLE_DEVICE) + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Device); + break; + #else + return false; + #endif + case axom::MemorySpace::Unified: + #if defined(UMPIRE_ENABLE_UM) + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Unified); + break; + #else + return false; + #endif + case axom::MemorySpace::Pinned: + #if defined(UMPIRE_ENABLE_PINNED) + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Pinned); + break; + #else + return false; + #endif + case axom::MemorySpace::Constant: + #if defined(UMPIRE_ENABLE_CONST) + axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Constant); + break; + #else + return false; + #endif + case axom::MemorySpace::Malloc: + case axom::MemorySpace::Dynamic: + break; + } + } + catch(const std::exception&) + { + return false; + } + + return true; +} +#endif + +template +int runtimeAllocatorIdForExecSpace() +{ +#if defined(AXOM_USE_UMPIRE) + if(axom::execution_space::onDevice()) + { + if(runtimeMemorySpaceAvailable(axom::MemorySpace::Device)) + { + return axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Device); + } + + if(runtimeMemorySpaceAvailable(axom::MemorySpace::Unified)) + { + return axom::getAllocatorIDFromMemorySpace(axom::MemorySpace::Unified); + } + + return axom::INVALID_ALLOCATOR_ID; + } + + return axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); +#else + return axom::getDefaultAllocatorID(); +#endif +} + } // namespace enum MeshType @@ -71,38 +149,9 @@ enum MeshType template struct ExecTraits { - static int getAllocatorId() - { -#ifdef AXOM_USE_UMPIRE - return axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Host); -#else - return axom::getDefaultAllocatorID(); -#endif - } + static int getAllocatorId() { return runtimeAllocatorIdForExecSpace(); } }; -#if defined(AXOM_RUNTIME_POLICY_USE_CUDA) -template -struct ExecTraits> -{ - static int getAllocatorId() - { - return axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Device); - } -}; -#endif - -#if defined(AXOM_RUNTIME_POLICY_USE_HIP) -template -struct ExecTraits> -{ - static int getAllocatorId() - { - return axom::getUmpireResourceAllocatorID(umpire::resource::MemoryResourceType::Device); - } -}; -#endif - /*! * Test fixture for PointInCell tests on MFEM meshes */ @@ -303,10 +352,16 @@ class PointInCellTest : public ::testing::Test template void testRandomPointsOnMesh(ExpectedValueFunctor exp, const std::string& meshTypeStr) { + const int hostAllocID = axom::execution_space::allocatorID(); + // Generate a PointInCell structure over the mesh axom::utilities::Timer constructTimer(true); // _quest_pic_init_start - PointInCellType spatialIndex(m_mesh, GridCell(25).data(), m_EPS, m_allocatorID); + PointInCellType spatialIndex(m_mesh, + GridCell(25).data(), + m_EPS, + m_allocatorID, + axom::HostAllocator {hostAllocID}); // _quest_pic_init_end SLIC_INFO( axom::fmt::format(axom::utilities::locale(), @@ -395,14 +450,18 @@ class PointInCellTest : public ::testing::Test /*! Tests PointInCell class using isoparametric points within each cell */ void testIsoGridPointsOnMesh(const std::string& meshTypeStr) { - int devAllocID = axom::execution_space::allocatorID(); + int devAllocID = m_allocatorID; int hostAllocID = axom::execution_space::allocatorID(); std::string filename = axom::fmt::format("quest_point_in_cell_{}_quad", meshTypeStr); // Add mesh to the grid axom::utilities::Timer constructTimer(true); - PointInCellType spatialIndex(m_mesh, GridCell(25).data(), m_EPS, m_allocatorID); + PointInCellType spatialIndex(m_mesh, + GridCell(25).data(), + m_EPS, + m_allocatorID, + axom::HostAllocator {hostAllocID}); SLIC_INFO( axom::fmt::format(axom::utilities::locale(), "Constructing index over {} quad mesh with {:L} elems took {:.3Lf} s", @@ -453,7 +512,6 @@ class PointInCellTest : public ::testing::Test // locate the reconstructed points (using EXEC space) if(axom::execution_space::onDevice()) { - int devAllocID = axom::execution_space::allocatorID(); // copy query points to device axom::Array spacePtsDevice(spacePts, devAllocID); @@ -552,6 +610,16 @@ class PointInCellTest : public ::testing::Test double getTolerance() const { return m_EPS; } + void skipIfExecAllocatorUnavailable() const + { + if(axom::execution_space::onDevice() && m_allocatorID == axom::INVALID_ALLOCATOR_ID) + { + GTEST_SKIP() << "Skipping test because no runtime-accessible device or unified allocator " + "is available for " + << axom::execution_space::name() << '.'; + } + } + protected: std::string m_meshDescriptorStr; @@ -578,6 +646,8 @@ class PointInCell2DTest : public PointInCellTest<2, ExecSpace> protected: virtual void SetUp() { + this->skipIfExecAllocatorUnavailable(); + /// Setup mesh strings, disable automatic formatting // clang-format off @@ -797,6 +867,8 @@ class PointInCell3DTest : public PointInCellTest<3, ExecSpace> protected: virtual void SetUp() { + this->skipIfExecAllocatorUnavailable(); + /// Setup mesh strings, disable automatic formatting // clang-format off diff --git a/src/axom/quest/tests/quest_signed_distance.cpp b/src/axom/quest/tests/quest_signed_distance.cpp index 71d42b855c..0cb6896927 100644 --- a/src/axom/quest/tests/quest_signed_distance.cpp +++ b/src/axom/quest/tests/quest_signed_distance.cpp @@ -297,9 +297,17 @@ void run_vectorized_sphere_test() { using PointType = primal::Point; - int host_allocator = axom::execution_space::allocatorID(); + const int current_allocator = axom::getDefaultAllocatorID(); + const axom::HostAllocator hostAllocator {axom::execution_space::allocatorID()}; int kernel_allocator = axom::execution_space::allocatorID(); +#if defined(AXOM_USE_UMPIRE) + if(!axom::execution_space::onDevice()) + { + kernel_allocator = axom::getUmpireResourceAllocatorID(umpire::resource::Host); + } +#endif + //Use unified memory on device #if defined(AXOM_USE_GPU) && defined(AXOM_USE_UMPIRE) if(axom::execution_space::onDevice()) @@ -352,7 +360,7 @@ void run_vectorized_sphere_test() double l2norm = 0.0; double linf = axom::numeric_limits::min(); - axom::Array queryPts = axom::Array(nnodes, nnodes, host_allocator); + axom::Array queryPts = axom::Array(nnodes, nnodes, hostAllocator.getID()); for(axom::IndexType inode = 0; inode < nnodes; ++inode) { umesh->getNode(inode, queryPts[inode].data()); @@ -403,7 +411,7 @@ void run_vectorized_sphere_test() delete surface_mesh; delete umesh; - axom::setDefaultAllocator(host_allocator); + axom::setDefaultAllocator(current_allocator); SLIC_INFO("Done."); } @@ -447,11 +455,12 @@ TEST(quest_signed_distance, sphere_vec_device_custom_alloc) using PointType = primal::Point; - const int host_allocator = axom::getUmpireResourceAllocatorID(umpire::resource::Host); + const int current_allocator = axom::getDefaultAllocatorID(); + const axom::HostAllocator hostAllocator {axom::execution_space::allocatorID()}; constexpr bool on_device = axom::execution_space::onDevice(); const int kernel_allocator = on_device ? axom::getUmpireResourceAllocatorID(umpire::resource::Unified) - : axom::execution_space::allocatorID(); + : axom::getUmpireResourceAllocatorID(umpire::resource::Host); axom::setDefaultAllocator(kernel_allocator); constexpr double l1norm_expected = 6.7051997372579715; @@ -521,7 +530,7 @@ TEST(quest_signed_distance, sphere_vec_device_custom_alloc) double l2norm = 0.0; double linf = axom::numeric_limits::min(); - axom::Array queryPts = axom::Array(nnodes, nnodes, host_allocator); + axom::Array queryPts = axom::Array(nnodes, nnodes, hostAllocator.getID()); for(axom::IndexType inode = 0; inode < nnodes; ++inode) { umesh->getNode(inode, queryPts[inode].data()); @@ -572,7 +581,7 @@ TEST(quest_signed_distance, sphere_vec_device_custom_alloc) delete surface_mesh; delete umesh; - axom::setDefaultAllocator(host_allocator); + axom::setDefaultAllocator(current_allocator); SLIC_INFO("Done."); } #endif // defined(AXOM_USE_GPU) && defined(AXOM_USE_RAJA) diff --git a/src/axom/quest/util/make_clipper_strategy.cpp b/src/axom/quest/util/make_clipper_strategy.cpp index 9eec1e0efd..8d5980daff 100644 --- a/src/axom/quest/util/make_clipper_strategy.cpp +++ b/src/axom/quest/util/make_clipper_strategy.cpp @@ -28,6 +28,13 @@ namespace util std::shared_ptr make_clipper_strategy(const axom::klee::Geometry& kleeGeometry, const std::string& name) +{ + return make_clipper_strategy(kleeGeometry, name, HostAllocator {}); +} + +std::shared_ptr make_clipper_strategy(const axom::klee::Geometry& kleeGeometry, + const std::string& name, + HostAllocator hostAllocator) { std::shared_ptr strategy; @@ -55,15 +62,15 @@ std::shared_ptr make_clipper_strategy(const axom::klee::Geo } else if(format == "sor3D") { - strategy.reset(new SORClipper(kleeGeometry, name)); + strategy.reset(new SORClipper(kleeGeometry, name, hostAllocator)); } else if(format == "cyl3D") { - strategy.reset(new MonotonicZSORClipper(kleeGeometry, name)); + strategy.reset(new MonotonicZSORClipper(kleeGeometry, name, hostAllocator)); } else if(format == "cone3D") { - strategy.reset(new MonotonicZSORClipper(kleeGeometry, name)); + strategy.reset(new MonotonicZSORClipper(kleeGeometry, name, hostAllocator)); } else { diff --git a/src/axom/quest/util/make_clipper_strategy.hpp b/src/axom/quest/util/make_clipper_strategy.hpp index aa9988063f..1d0cd6d042 100644 --- a/src/axom/quest/util/make_clipper_strategy.hpp +++ b/src/axom/quest/util/make_clipper_strategy.hpp @@ -11,6 +11,7 @@ // MeshClipper depends on sidre #ifdef AXOM_USE_SIDRE + #include "axom/core/memory_management.hpp" #include "axom/klee/Geometry.hpp" #include "axom/quest/MeshClipperStrategy.hpp" @@ -34,12 +35,32 @@ namespace util * klee geometry formats. It issues a warning for unrecognized * formats. * + * \note This compatibility overload uses Axom's current default host allocator + * for any constructor-time host scratch. Prefer the overload that accepts + * `HostAllocator` when host allocator ownership is available. + * * @return Pointer to new MeshClipperStrategy, or null if the * specified geometry is not an axom-provided one. */ std::shared_ptr make_clipper_strategy(const axom::klee::Geometry& kleeGeometry, const std::string& name = ""); +/*! + * @brief Return a new MeshClipperStrategy implementation using an explicit + * host allocator for constructor-time host scratch and storage in strategies + * that need it. + * + * @param [in] kleeGeometry Geometry description. + * @param [in] name Name of strategy instance. + * @param [in] hostAllocator Allocator for host-resident strategy construction. + * + * @return Pointer to new MeshClipperStrategy, or null if the + * specified geometry is not an axom-provided one. + */ +std::shared_ptr make_clipper_strategy(const axom::klee::Geometry& kleeGeometry, + const std::string& name, + HostAllocator hostAllocator); + } // namespace util } // namespace experimental } // namespace quest diff --git a/src/axom/sidre/tests/sidre_group.cpp b/src/axom/sidre/tests/sidre_group.cpp index d2dc061fe9..c95334ba5d 100644 --- a/src/axom/sidre/tests/sidre_group.cpp +++ b/src/axom/sidre/tests/sidre_group.cpp @@ -1479,7 +1479,7 @@ std::vector getKnownAllocIds() { std::vector allocIds(1, axom::MALLOC_ALLOCATOR_ID); #ifdef AXOM_USE_UMPIRE - allocIds.push_back(axom::detail::getAllocatorID()); + allocIds.push_back(axom::HostAllocator {}.getID()); #ifdef AXOM_USE_GPU allocIds.push_back(axom::detail::getAllocatorID()); allocIds.push_back(axom::detail::getAllocatorID()); @@ -3583,24 +3583,10 @@ TEST(sidre_group, import_conduit_lists) //------------------------------------------------------------------------------ -inline int pointerToAllocatorID(const void* ptr) -{ -#ifdef AXOM_USE_UMPIRE - umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); - if(rm.hasAllocator(const_cast(ptr))) - { - umpire::Allocator allocator = rm.getAllocator(const_cast(ptr)); - return allocator.getId(); - } -#endif - AXOM_UNUSED_VAR(ptr); - return axom::getDefaultAllocatorID(); -} - TEST(sidre_group, import_conduit_into_mem_space) { #if defined(AXOM_USE_UMPIRE) && defined(UMPIRE_ENABLE_DEVICE) - const int hostAllocId = axom::detail::getAllocatorID(); + const int hostAllocId = axom::HostAllocator {}.getID(); const int devAllocId = axom::detail::getAllocatorID(); #else // Memory space testing is trivial without device memory. @@ -3618,8 +3604,9 @@ TEST(sidre_group, import_conduit_into_mem_space) conduit::Node node; node["origOnHost"].set_dtype(dtype); node["origOnDev"].set_external(dtype, devArray); - EXPECT_EQ(pointerToAllocatorID(node["origOnHost"].data_ptr()), hostAllocId); - EXPECT_EQ(pointerToAllocatorID(node["origOnDev"].data_ptr()), devAllocId); + EXPECT_TRUE(axom::isHostAccessibleAllocatorID( + axom::getAllocatorIDFromPointer(node["origOnHost"].data_ptr()))); + EXPECT_EQ(axom::getAllocatorIDFromPointer(node["origOnDev"].data_ptr()), devAllocId); /* Regardless of where the source data is, the destination @@ -3638,11 +3625,11 @@ TEST(sidre_group, import_conduit_into_mem_space) EXPECT_TRUE(destGroup->hasView("origOnDev")); const auto* viewWithHostData = destGroup->getView("origOnHost"); - int allocIdForHostData = pointerToAllocatorID(viewWithHostData->getVoidPtr()); + int allocIdForHostData = axom::getAllocatorIDFromPointer(viewWithHostData->getVoidPtr()); EXPECT_EQ(allocIdForHostData, destAllocId); const auto* viewWithDevData = destGroup->getView("origOnDev"); - int allocIdForDevData = pointerToAllocatorID(viewWithDevData->getVoidPtr()); + int allocIdForDevData = axom::getAllocatorIDFromPointer(viewWithDevData->getVoidPtr()); EXPECT_EQ(allocIdForDevData, destAllocId); } @@ -3658,11 +3645,11 @@ TEST(sidre_group, import_conduit_into_mem_space) EXPECT_TRUE(destGroup->hasView("origOnDev")); const auto* viewWithHostData = destGroup->getView("origOnHost"); - int allocIdForHostData = pointerToAllocatorID(viewWithHostData->getVoidPtr()); + int allocIdForHostData = axom::getAllocatorIDFromPointer(viewWithHostData->getVoidPtr()); EXPECT_EQ(allocIdForHostData, destAllocId); const auto* viewWithDevData = destGroup->getView("origOnDev"); - int allocIdForDevData = pointerToAllocatorID(viewWithDevData->getVoidPtr()); + int allocIdForDevData = axom::getAllocatorIDFromPointer(viewWithDevData->getVoidPtr()); EXPECT_EQ(allocIdForDevData, destAllocId); } diff --git a/src/axom/sidre/tests/sidre_view.cpp b/src/axom/sidre/tests/sidre_view.cpp index 6224ff3e3b..d65181a95c 100644 --- a/src/axom/sidre/tests/sidre_view.cpp +++ b/src/axom/sidre/tests/sidre_view.cpp @@ -7,6 +7,7 @@ #include "gtest/gtest.h" #include "axom/config.hpp" #include "axom/core/Types.hpp" +#include "axom/core/memory_management.hpp" #include "axom/core/utilities/FileUtilities.hpp" #include "axom/sidre/core/ConduitMemory.hpp" #include "axom/slic.hpp" @@ -1992,7 +1993,7 @@ std::vector getKnownAllocIds() { std::vector allocIds(1, axom::MALLOC_ALLOCATOR_ID); #ifdef AXOM_USE_UMPIRE - allocIds.push_back(axom::detail::getAllocatorID()); + allocIds.push_back(axom::HostAllocator {}.getID()); #ifdef AXOM_USE_GPU allocIds.push_back(axom::detail::getAllocatorID()); allocIds.push_back(axom::detail::getAllocatorID()); diff --git a/src/axom/spin/BVH.hpp b/src/axom/spin/BVH.hpp index 8cd7552c44..c2f057ea1f 100644 --- a/src/axom/spin/BVH.hpp +++ b/src/axom/spin/BVH.hpp @@ -11,6 +11,7 @@ #include "axom/core/Macros.hpp" // for Axom macros #include "axom/core/Types.hpp" // for axom::IndexType #include "axom/core/numerics/floating_point_limits.hpp" // floating_point_limits +#include "axom/core/memory_management.hpp" // for HostAllocator #include "axom/core/execution/execution_space.hpp" // for execution spaces @@ -327,6 +328,9 @@ class BVH * \param [in] numPts the total number of query points supplied * \param [in] points array of points to query against the BVH * + * \note The overload without `HostAllocator` is a compatibility path that + * uses Axom's current default host allocator for candidate host staging. + * * \note Upon completion, the ith query point has: * * counts[ i ] candidates * * Stored in the candidates array in the following range: @@ -344,6 +348,15 @@ class BVH axom::Array& candidates, IndexType numPts, PointIndexable points) const; + /// \overload + /// \param [in] hostAllocator allocator for candidate host staging. + template + void findPoints(axom::ArrayView offsets, + axom::ArrayView counts, + axom::Array& candidates, + IndexType numPts, + PointIndexable points, + HostAllocator hostAllocator) const; /*! * \brief Finds the candidate bins that intersect the given rays. @@ -354,6 +367,9 @@ class BVH * \param [in] numRays the total number of rays * \param [in] rays array of the rays to query against the BVH * + * \note The overload without `HostAllocator` is a compatibility path that + * uses Axom's current default host allocator for candidate host staging. + * * \note After the call to findRays(), the ith ray has: * * counts[ i ] candidates * * candidates stored in [ offsets[ i ], offsets[i]+counts[i] ] @@ -370,6 +386,15 @@ class BVH axom::Array& candidates, IndexType numRays, RayIndexable rays) const; + /// \overload + /// \param [in] hostAllocator allocator for candidate host staging. + template + void findRays(axom::ArrayView offsets, + axom::ArrayView counts, + axom::Array& candidates, + IndexType numRays, + RayIndexable rays, + HostAllocator hostAllocator) const; /*! * \brief Finds the candidate bins that intersect the given bounding boxes. @@ -380,6 +405,9 @@ class BVH * \param [in] numBoxes the total number of bounding boxes * \param [in] boxes array of boxes to query against the BVH * + * \note The overload without `HostAllocator` is a compatibility path that + * uses Axom's current default host allocator for candidate host staging. + * * \note After the call to findBoundingBoxes(), the ith bounding box has: * * counts[ i ] candidates * * candidates stored in [ offsets[ i ], offsets[i]+counts[i] ] @@ -396,6 +424,15 @@ class BVH axom::Array& candidates, IndexType numBoxes, BoxIndexable boxes) const; + /// \overload + /// \param [in] hostAllocator allocator for candidate host staging. + template + void findBoundingBoxes(axom::ArrayView offsets, + axom::ArrayView counts, + axom::Array& candidates, + IndexType numBoxes, + BoxIndexable boxes, + HostAllocator hostAllocator) const; /*! * \brief Writes the BVH to the specified VTK file for visualization. @@ -484,6 +521,19 @@ void BVH::findPoints(axom::ArrayView& candidates, IndexType numPts, PointIndexable pts) const +{ + findPoints(offsets, counts, candidates, numPts, pts, HostAllocator {}); +} + +//------------------------------------------------------------------------------ +template +template +void BVH::findPoints(axom::ArrayView offsets, + axom::ArrayView counts, + axom::Array& candidates, + IndexType numPts, + PointIndexable pts, + HostAllocator hostAllocator) const { AXOM_ANNOTATE_SCOPE("BVH::findPoints"); @@ -500,8 +550,13 @@ void BVH::findPoints(axom::ArrayViewtemplate findCandidatesImpl(predicate, offsets, counts, numPts, pts, m_AllocatorID); + candidates = m_bvh->template findCandidatesImpl(predicate, + offsets, + counts, + numPts, + pts, + m_AllocatorID, + hostAllocator); } //------------------------------------------------------------------------------ @@ -512,6 +567,19 @@ void BVH::findRays(axom::ArrayView axom::Array& candidates, IndexType numRays, RayIndexable rays) const +{ + findRays(offsets, counts, candidates, numRays, rays, HostAllocator {}); +} + +//------------------------------------------------------------------------------ +template +template +void BVH::findRays(axom::ArrayView offsets, + axom::ArrayView counts, + axom::Array& candidates, + IndexType numRays, + RayIndexable rays, + HostAllocator hostAllocator) const { AXOM_ANNOTATE_SCOPE("BVH::findRays"); @@ -531,8 +599,13 @@ void BVH::findRays(axom::ArrayView return primal::detail::intersect_ray(r, bb, tmp, TOL); }; - candidates = - m_bvh->template findCandidatesImpl(predicate, offsets, counts, numRays, rays, m_AllocatorID); + candidates = m_bvh->template findCandidatesImpl(predicate, + offsets, + counts, + numRays, + rays, + m_AllocatorID, + hostAllocator); } //------------------------------------------------------------------------------ @@ -543,6 +616,19 @@ void BVH::findBoundingBoxes(axom::ArrayView& candidates, IndexType numBoxes, BoxIndexable boxes) const +{ + findBoundingBoxes(offsets, counts, candidates, numBoxes, boxes, HostAllocator {}); +} + +//------------------------------------------------------------------------------ +template +template +void BVH::findBoundingBoxes(axom::ArrayView offsets, + axom::ArrayView counts, + axom::Array& candidates, + IndexType numBoxes, + BoxIndexable boxes, + HostAllocator hostAllocator) const { AXOM_ANNOTATE_SCOPE("BVH::findBoundingBoxes"); @@ -564,7 +650,8 @@ void BVH::findBoundingBoxes(axom::ArrayView outOffsets, axom::ArrayView outCounts, - axom::Array& outCandidates) const; + axom::Array& outCandidates) const + { + getCandidatesAsArray(qsize, queryObjs, outOffsets, outCounts, outCandidates, HostAllocator {}); + } + + /// \overload + /// \param [in] hostAllocator allocator for host staging and fallback. + template + void getCandidatesAsArray(axom::IndexType qsize, + const QueryGeom* queryObjs, + axom::ArrayView outOffsets, + axom::ArrayView outCounts, + axom::Array& outCandidates, + HostAllocator hostAllocator) const; /// \overload void getCandidatesAsArray(axom::ArrayView queryObjs, @@ -470,7 +486,27 @@ class ImplicitGrid axom::ArrayView outCounts, axom::Array& outCandidates) const { - getCandidatesAsArray(queryObjs.size(), queryObjs.data(), outOffsets, outCounts, outCandidates); + getCandidatesAsArray(queryObjs.size(), + queryObjs.data(), + outOffsets, + outCounts, + outCandidates, + HostAllocator {}); + } + + /// \overload + void getCandidatesAsArray(axom::ArrayView queryObjs, + axom::ArrayView outOffsets, + axom::ArrayView outCounts, + axom::Array& outCandidates, + HostAllocator hostAllocator) const + { + getCandidatesAsArray(queryObjs.size(), + queryObjs.data(), + outOffsets, + outCounts, + outCandidates, + hostAllocator); } /// \overload @@ -479,7 +515,27 @@ class ImplicitGrid axom::ArrayView outCounts, axom::Array& outCandidates) const { - getCandidatesAsArray(queryObjs.size(), queryObjs.data(), outOffsets, outCounts, outCandidates); + getCandidatesAsArray(queryObjs.size(), + queryObjs.data(), + outOffsets, + outCounts, + outCandidates, + HostAllocator {}); + } + + /// \overload + void getCandidatesAsArray(axom::ArrayView queryObjs, + axom::ArrayView outOffsets, + axom::ArrayView outCounts, + axom::Array& outCandidates, + HostAllocator hostAllocator) const + { + getCandidatesAsArray(queryObjs.size(), + queryObjs.data(), + outOffsets, + outCounts, + outCandidates, + hostAllocator); } /*! @@ -811,7 +867,8 @@ void ImplicitGrid::getCandidatesAsArray( const QueryGeom* queryObjs, axom::ArrayView outOffsets, axom::ArrayView outCounts, - axom::Array& outCandidates) const + axom::Array& outCandidates, + HostAllocator hostAllocator) const { SLIC_ERROR_IF(outOffsets.size() < qsize, "outOffsets must have at least qsize elements"); SLIC_ERROR_IF(outCounts.size() < qsize, "outCounts must have at least qsize elements"); @@ -833,7 +890,7 @@ void ImplicitGrid::getCandidatesAsArray( axom::IndexType totalCount = totalCountReduce.get(); // Step 3: allocate memory for all candidates - outCandidates = axom::Array(totalCount, totalCount, m_allocatorId); + outCandidates = axom::Array(totalCount, totalCount, m_allocatorId, hostAllocator); const auto candidates_v = outCandidates.view(); // Step 4: fill candidate array for each query box @@ -851,6 +908,7 @@ void ImplicitGrid::getCandidatesAsArray( gridQuery.visitCandidates(queryObjs[i], onCandidate); }); #else + outCandidates = axom::Array(0, 0, m_allocatorId, hostAllocator); outOffsets[0] = 0; for(int i = 0; i < qsize; i++) { diff --git a/src/axom/spin/UniformGrid.hpp b/src/axom/spin/UniformGrid.hpp index c3b6a90c11..8c6b17f643 100644 --- a/src/axom/spin/UniformGrid.hpp +++ b/src/axom/spin/UniformGrid.hpp @@ -183,6 +183,8 @@ class UniformGrid : StoragePolicy * * \note The output candidate array is allocated inside the function, using * the given allocator ID passed in during implicit grid initialization. + * The overload without `HostAllocator` is a compatibility path that uses + * Axom's current default host allocator for host staging and fallback. * * \note Upon completion, the ith query point has: * * counts[ i ] candidates @@ -194,6 +196,14 @@ class UniformGrid : StoragePolicy axom::ArrayView outCounts, axom::Array& outCandidates) const; + /// \overload + /// \param [in] hostAllocator allocator for host staging and fallback. + void getCandidatesAsArray(axom::ArrayView queryObjs, + axom::ArrayView outOffsets, + axom::ArrayView outCounts, + axom::Array& outCandidates, + HostAllocator hostAllocator) const; + /*! * \brief Clears the bin indicated by index. * @@ -696,6 +706,18 @@ void UniformGrid::getCandidatesAsArray( axom::ArrayView outOffsets, axom::ArrayView outCounts, axom::Array& outCandidates) const +{ + getCandidatesAsArray(queryObjs, outOffsets, outCounts, outCandidates, HostAllocator {}); +} + +//------------------------------------------------------------------------------ +template +void UniformGrid::getCandidatesAsArray( + axom::ArrayView queryObjs, + axom::ArrayView outOffsets, + axom::ArrayView outCounts, + axom::Array& outCandidates, + HostAllocator hostAllocator) const { IndexType qsize = queryObjs.size(); SLIC_ASSERT(qsize > 0); @@ -724,8 +746,9 @@ void UniformGrid::getCandidatesAsArray( axom::IndexType totalCount = totalCountReduce.get(); // Step 3: allocate memory for all candidates - axom::Array queryIndex(totalCount, totalCount, this->getAllocatorID()); - outCandidates = axom::Array(totalCount, totalCount, this->getAllocatorID()); + axom::Array queryIndex(totalCount, totalCount, this->getAllocatorID(), hostAllocator); + outCandidates = + axom::Array(totalCount, totalCount, this->getAllocatorID(), hostAllocator); const auto query_idx_view = queryIndex.view(); const auto candidates_view = outCandidates.view(); @@ -774,7 +797,7 @@ void UniformGrid::getCandidatesAsArray( // Step 6: Count and flag unique intersection pairs, in order to map them // to a deduplicated candidate intersection array. axom::ReduceSum dedupCountReduce(0); - axom::Array dedupTgtIdx(totalCount, totalCount, this->getAllocatorID()); + axom::Array dedupTgtIdx(totalCount, totalCount, this->getAllocatorID(), hostAllocator); const auto dedup_idx_view = dedupTgtIdx.view(); for_all( totalCount, @@ -802,7 +825,7 @@ void UniformGrid::getCandidatesAsArray( // Step 7: Fill the array of deduplicated candidates based on the index // mapping generated previously. axom::IndexType dedupSize = dedupCountReduce.get(); - axom::Array dedupedCandidates(dedupSize, dedupSize, this->getAllocatorID()); + axom::Array dedupedCandidates(dedupSize, dedupSize, this->getAllocatorID(), hostAllocator); const auto dedup_cand_view = dedupedCandidates.view(); // Reset counts counter for counting unique candidates per query box. @@ -834,6 +857,7 @@ void UniformGrid::getCandidatesAsArray( outCandidates = std::move(dedupedCandidates); #else // AXOM_USE_RAJA + outCandidates = axom::Array(0, 0, this->getAllocatorID(), hostAllocator); outOffsets[0] = 0; for(int i = 0; i < qsize; i++) { diff --git a/src/axom/spin/docs/sphinx/bvh.rst b/src/axom/spin/docs/sphinx/bvh.rst index 343b58c084..8875df8d70 100644 --- a/src/axom/spin/docs/sphinx/bvh.rst +++ b/src/axom/spin/docs/sphinx/bvh.rst @@ -42,6 +42,11 @@ probe point must be tested against each triangle. Note that the returned packed candidate intersection array (``candidatesPtr`` above) needs to be deallocated by the caller. +When using candidate-query overloads that allocate ``axom::Array`` output, pass +an ``axom::HostAllocator`` when host staging or host fallback allocation should +be controlled explicitly. Query overloads without a host allocator remain +available as compatibility paths and use Axom's current default host allocator. + Finally, test the point against all candidate neighbor triangles. .. literalinclude:: ../../examples/spin_introduction.cpp diff --git a/src/axom/spin/docs/sphinx/implicitgrid.rst b/src/axom/spin/docs/sphinx/implicitgrid.rst index 463cfbef19..290862c01e 100644 --- a/src/axom/spin/docs/sphinx/implicitgrid.rst +++ b/src/axom/spin/docs/sphinx/implicitgrid.rst @@ -38,9 +38,13 @@ After including the header and setting up types, set up the index. Inexpensive queries to the index reduce the number of calls to a (possibly) expensive test routine. +Candidate-output APIs that allocate or stage host data have overloads that +accept ``axom::HostAllocator``. Prefer those overloads in new code when host +allocator ownership is available; overloads without a host allocator are +compatibility paths that use Axom's current default host allocator. + .. literalinclude:: ../../examples/spin_introduction.cpp :start-after: _igrid_query_start :end-before: _igrid_query_end :language: C++ - diff --git a/src/axom/spin/docs/sphinx/uniformgrid.rst b/src/axom/spin/docs/sphinx/uniformgrid.rst index 0bcc2117bd..f1ddc3c5bb 100644 --- a/src/axom/spin/docs/sphinx/uniformgrid.rst +++ b/src/axom/spin/docs/sphinx/uniformgrid.rst @@ -46,6 +46,11 @@ First, construct the ``UniformGrid`` and load it with triangles. Then, for every triangle, look up its possible neighbors +Candidate-output APIs that allocate or stage host data have overloads that +accept ``axom::HostAllocator``. Prefer those overloads in new code when host +allocator ownership is available; overloads without a host allocator are +compatibility paths that use Axom's current default host allocator. + .. literalinclude:: ../../examples/spin_introduction.cpp :start-after: _ugrid_candidate_start :end-before: _ugrid_candidate_end @@ -61,4 +66,3 @@ and test the triangle against those neighbors. The ``UniformGrid`` has its best effect when objects are roughly the same size and evenly distributed over the region of interest, and when bins are close to the characteristic size of objects in the region of interest. - diff --git a/src/axom/spin/policy/LinearBVH.hpp b/src/axom/spin/policy/LinearBVH.hpp index 7db3f14ca5..0568c90607 100644 --- a/src/axom/spin/policy/LinearBVH.hpp +++ b/src/axom/spin/policy/LinearBVH.hpp @@ -142,9 +142,35 @@ class LinearBVHTraverser template axom::Array reduce_tree(LeafAction&& leaf_action, int allocatorID = axom::getDefaultAllocatorID()) const + { + const HostAllocator hostAllocator = axom::detail::hostAllocatorForPrimaryAllocator(allocatorID); + return reduce_tree(std::forward(leaf_action), + allocatorID, + hostAllocator); + } + + /*! + * \brief Iterate over the tree, invoking the leaf action at each leaf node to + * produce a value and then iterate back up the tree, combining nodes + * using a "+" reduction. Return the Array that contains values for + * all tree nodes. + * + * \param leaf_action The function to invoke on a leaf node to make its data. + * \param allocatorID The allocator to use to allocate primary array data. + * \param hostAllocator Allocator to use for host-accessible staging/scratch. + * + * \return An Array that contains the reduced data for all nodes in the BVH. + */ + template + axom::Array reduce_tree(LeafAction&& leaf_action, + int allocatorID, + HostAllocator hostAllocator) const { // Make a field over all of the nodes (the return field). - axom::Array reducedField(m_inner_nodes.size(), m_inner_nodes.size(), allocatorID); + axom::Array reducedField(m_inner_nodes.size(), + m_inner_nodes.size(), + allocatorID, + hostAllocator); if constexpr(std::is_same_v) { @@ -156,7 +182,10 @@ class LinearBVHTraverser // Do it in 2 stages. // Make a field for just the leaf data. Compute it in parallel. - axom::Array leafField(m_leaf_nodes.size(), m_leaf_nodes.size(), allocatorID); + axom::Array leafField(m_leaf_nodes.size(), + m_leaf_nodes.size(), + allocatorID, + hostAllocator); auto leafFieldView = leafField.view(); const std::int32_t* leaf_nodes_data = m_leaf_nodes.data(); axom::for_all(m_leaf_nodes.size(), [&](axom::IndexType currentNode) { @@ -261,7 +290,8 @@ class LinearBVH const axom::ArrayView counts, IndexType numObjs, PrimitiveIndexable objs, - int allocatorID) const; + int allocatorID, + HostAllocator hostAllocator) const; void writeVtkFileImpl(const std::string& fileName) const; @@ -390,7 +420,8 @@ axom::Array LinearBVH::findCandidatesImp const axom::ArrayView counts, IndexType numObjs, PrimitiveIndexable objs, - int allocatorID) const + int allocatorID, + HostAllocator hostAllocator) const { AXOM_ANNOTATE_SCOPE("LinearBVH::findCandidatesImpl"); @@ -444,7 +475,8 @@ axom::Array LinearBVH::findCandidatesImp // STEP 3: allocate memory for all candidates AXOM_ANNOTATE_BEGIN("allocate_candidates"); - auto candidates = axom::Array(total_candidates, total_candidates, allocatorID); + auto candidates = + axom::Array(total_candidates, total_candidates, allocatorID, hostAllocator); AXOM_ANNOTATE_END("allocate_candidates"); const auto candidates_v = candidates.view(); @@ -475,7 +507,7 @@ axom::Array LinearBVH::findCandidatesImp #else // CPU-only and no RAJA: do single traversal AXOM_UNUSED_VAR(allocatorID); - axom::Array search_candidates; + axom::Array search_candidates(0, 0, hostAllocator.getID(), hostAllocator); int current_offset = 0; // STEP 1: do single-pass traversal with std::vector for candidates diff --git a/src/axom/spin/tests/spin_bvh.cpp b/src/axom/spin/tests/spin_bvh.cpp index bb459c9bdf..3fd5f680ca 100644 --- a/src/axom/spin/tests/spin_bvh.cpp +++ b/src/axom/spin/tests/spin_bvh.cpp @@ -7,6 +7,7 @@ // axom includes #include "axom/config.hpp" #include "axom/core.hpp" +#include "axom/core/utilities/MemoryTesting.hpp" #include "axom/primal.hpp" #include "axom/mint.hpp" @@ -1175,7 +1176,7 @@ void check_build_bvh_zip3d() using ZipIter = typename primal::ZipIndexable; const int current_allocator = axom::getDefaultAllocatorID(); - axom::setDefaultAllocator(axom::execution_space::allocatorID()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); FloatType* xmin = axom::allocate(NUM_BOXES); FloatType* ymin = axom::allocate(NUM_BOXES); @@ -1242,7 +1243,7 @@ void check_find_points_zip3d() constexpr IndexType N = 4; const int current_allocator = axom::getDefaultAllocatorID(); - axom::setDefaultAllocator(axom::execution_space::allocatorID()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); using BoxType = typename primal::BoundingBox; using PointType = primal::Point; @@ -1329,7 +1330,7 @@ void check_find_points_zip2d() constexpr IndexType N = 4; const int current_allocator = axom::getDefaultAllocatorID(); - axom::setDefaultAllocator(axom::execution_space::allocatorID()); + axom::setDefaultAllocator(axom::utilities::globalDefaultAllocatorForExecSpace()); using BoxType = typename primal::BoundingBox; using PointType = primal::Point; diff --git a/src/docs/sphinx/dev_guide/gpu_porting.rst b/src/docs/sphinx/dev_guide/gpu_porting.rst index 3f448caafe..baa8141fda 100644 --- a/src/docs/sphinx/dev_guide/gpu_porting.rst +++ b/src/docs/sphinx/dev_guide/gpu_porting.rst @@ -125,6 +125,13 @@ the memory space where data in ``Dynamic`` allows you to define the location at run time, with some caveats (see :ref:`Core Containers` for more details and examples). +``Host`` refers to Axom's current default CPU allocator. By default this is the +platform host allocator, but it can also be configured to use Axom's +malloc-backed allocator with ``axom::setDefaultHostAllocator()``. + +``Dynamic`` remains separate from ``Host``. In Umpire-enabled builds, +``Dynamic`` continues to follow the current default Umpire allocator. + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Useful Links ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -158,8 +165,7 @@ GPU device, or on both a GPU device and a CPU host. For example:: .. note:: - When Axom is built without RAJA, ``axom::for_all`` becomes a ``for``-loop on - host (CPU). + When Axom is built without RAJA, ``axom::for_all`` becomes a for-loop on host (CPU). %%%%%%%%%%%%%%%% Portability @@ -168,8 +174,8 @@ Portability Adherence to the GPU porting guidelines generally results in code that will compile and run on multiple backends. However, backends such as CUDA require additional guidelines. -**Do not use ``auto`` lambda parameters** with ``axom::for_all`` or the code will not -compile under nvcc. +**Do not use auto lambda parameters** with ``axom::for_all`` or the code will not +compile with nvcc. Do this: @@ -211,13 +217,14 @@ Do NOT do this: .. code-block:: cpp + /* Here, dataView is a reference. This causes the error described in the for_all body. */ template void doSomething(axom::ArrayView &dataView) { axom::for_all(dataView.size(), AXOM_LAMBDA(axom::IndexType index) { /* body uses dataView[index] */ - /* It will crash on GPU devices because the host reference was + /* This will cause an error on a GPU because the host reference is captured rather than the object. */ });