diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 66817da9bc..cc3559429c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -18,6 +18,7 @@ set(REALM_SOURCES network.cc codedesc.cc logging.cc + loader.cc mutex.cc profiling.cc timers.cc diff --git a/src/realm/cuda/cuda_internal.h b/src/realm/cuda/cuda_internal.h index dcc32a12fd..245cb0637d 100644 --- a/src/realm/cuda/cuda_internal.h +++ b/src/realm/cuda/cuda_internal.h @@ -19,6 +19,7 @@ #define REALM_CUDA_INTERNAL_H #include "realm/cuda/cuda_module.h" +#include "realm/loader.h" #include #include @@ -67,8 +68,8 @@ #define REPORT_CU_ERROR(level, cmd, ret) \ do { \ const char *name, *str; \ - CUDA_DRIVER_FNPTR(Realm::Cuda::cuGetErrorName)(ret, &name); \ - CUDA_DRIVER_FNPTR(Realm::Cuda::cuGetErrorString)(ret, &str); \ + CUDA_DRIVER_FNPTR(cuGetErrorName)(ret, &name); \ + CUDA_DRIVER_FNPTR(cuGetErrorString)(ret, &str); \ log_gpu.newmsg(level) << __FILE__ << '(' << __LINE__ << "):" << cmd << " = " << ret \ << '(' << name << "): " << str; \ } while(0) @@ -1292,9 +1293,15 @@ namespace Realm { #endif #endif - // cuda driver and/or runtime entry points -#define CUDA_DRIVER_HAS_FNPTR(name) ((name##_fnptr) != nullptr) -#define CUDA_DRIVER_FNPTR(name) (assert(name##_fnptr != nullptr), name##_fnptr) +// cuda driver and/or runtime entry points +#if defined(REALM_CUDA_DYNAMIC_LOAD) +#define CUDA_DRIVER_HAS_FNPTR(name) ((cuda_loader.name##_fnptr) != nullptr) +#define CUDA_DRIVER_FNPTR(name) \ + (assert(cuda_loader.name##_fnptr != nullptr), cuda_loader.name##_fnptr) +#else +#define CUDA_DRIVER_HAS_FNPTR(name) ((name) != nullptr) +#define CUDA_DRIVER_FNPTR(name) (assert(name != nullptr), name) +#endif // Only APIs that are available in the minimum base driver version that Realm supports // should be listed here @@ -1410,12 +1417,17 @@ namespace Realm { __op__(cuCtxRecordEvent, 12050); \ __op__(cuArrayGetMemoryRequirements, CUDA_VERSION_MIN); + struct CudaLoader : public Loader { // Make sure to only use decltype, to ensure it matches the cuda.h definition -#define DECL_FNPTR_EXTERN(name, ver) extern decltype(&name) name##_fnptr; - CUDA_DRIVER_APIS(DECL_FNPTR_EXTERN); -#undef DECL_FNPTR_EXTERN +#define DECL_FNPTR(name, ver) decltype(&name) name##_fnptr = nullptr; + CUDA_DRIVER_APIS(DECL_FNPTR); +#undef DECL_FNPTR + bool load_symbols(); + }; -#define NVML_FNPTR(name) (name##_fnptr) + extern CudaLoader cuda_loader; + +#define NVML_FNPTR(name) (nvml_loader.name##_fnptr) #if NVML_API_VERSION >= 11 #define NVML_11_APIS(__op__) __op__(nvmlDeviceGetMemoryAffinity); @@ -1464,6 +1476,16 @@ namespace Realm { NVML_11_APIS(__op__); \ NVML_12_APIS(__op__); + struct NVMLLoader : public Loader { +// Make sure to only use decltype, to ensure it matches the cuda.h definition +#define DECL_FNPTR(name) decltype(&name) name##_fnptr = nullptr; + NVML_APIS(DECL_FNPTR); +#undef DECL_FNPTR + bool load_symbols(); + }; + + extern NVMLLoader nvml_loader; + #define DECL_FNPTR_EXTERN(name) extern decltype(&name) name##_fnptr; NVML_APIS(DECL_FNPTR_EXTERN) #undef DECL_FNPTR_EXTERN @@ -1484,8 +1506,19 @@ namespace Realm { CUPTI_APIS(DECL_FNPTR_EXTERN) #undef DECL_FNPTR_EXTERN -#define CUPTI_HAS_FNPTR(name) (name##_fnptr != nullptr) -#define CUPTI_FNPTR(name) (assert(name##_fnptr != nullptr), name##_fnptr) +#define CUPTI_HAS_FNPTR(name) (cupti_loader.name##_fnptr != nullptr) +#define CUPTI_FNPTR(name) \ + (assert(cupti_loader.name##_fnptr != nullptr), cupti_loader.name##_fnptr) + + struct CUPTILoader : public Loader { +// Make sure to only use decltype, to ensure it matches the cuda.h definition +#define DECL_FNPTR(name) decltype(&name) name##_fnptr = nullptr; + CUPTI_APIS(DECL_FNPTR); +#undef DECL_FNPTR + bool load_symbols(); + }; + + extern CUPTILoader cupti_loader; }; // namespace Cuda diff --git a/src/realm/cuda/cuda_module.cc b/src/realm/cuda/cuda_module.cc index 176fbc57ab..62bad98dcf 100644 --- a/src/realm/cuda/cuda_module.cc +++ b/src/realm/cuda/cuda_module.cc @@ -98,6 +98,9 @@ namespace Realm { Logger log_cudart("cudart"); Logger log_cudaipc("cudaipc"); Logger log_cupti("cupti"); + CudaLoader cuda_loader; + NVMLLoader nvml_loader; + CUPTILoader cupti_loader; Logger log_stream("gpustream"); bool nvml_api_fnptrs_loaded = false; @@ -106,14 +109,6 @@ namespace Realm { bool cupti_api_initialized = false; CUresult cuda_init_code = CUDA_ERROR_UNKNOWN; - bool cuda_api_fnptrs_loaded = false; - -// Make sure to only use decltype here, to ensure it matches the cuda.h definition -#define DEFINE_FNPTR(name, ver) decltype(&name) name##_fnptr = 0; - - CUDA_DRIVER_APIS(DEFINE_FNPTR); -#undef DEFINE_FNPTR - static unsigned ctz(uint64_t v) { #ifdef REALM_ON_WINDOWS @@ -136,12 +131,6 @@ namespace Realm { #endif } -#define DEFINE_FNPTR(name) decltype(&name) name##_fnptr = 0; - - NVML_APIS(DEFINE_FNPTR); - CUPTI_APIS(DEFINE_FNPTR); -#undef DEFINE_FNPTR - // function pointers for cuda hook typedef void (*PFN_cuhook_register_callback)(void); typedef void (*PFN_cuhook_start_task)(CUstream current_task_stream); @@ -2694,128 +2683,87 @@ namespace Realm { } } - static bool resolve_cuda_api_fnptrs(void) + bool CudaLoader::load_symbols() { - if(cuda_api_fnptrs_loaded) { - return true; - } - decltype(&cuGetProcAddress) cuGetProcAddress_fnptr = nullptr; - -#if defined(REALM_USE_LIBDL) - log_gpu.info() << "dynamically loading libcuda.so"; - void *libcuda = dlopen("libcuda.so.1", RTLD_NOW); - if(!libcuda) { - log_gpu.info() << "could not open libcuda.so: " << strerror(errno); - return false; - } - // Use the symbol we get from the dynamically loaded library - cuGetProcAddress_fnptr = reinterpret_cast( - dlsym(libcuda, STRINGIFY(cuGetProcAddress))); -#elif CUDA_VERSION >= 11030 - // Use the statically available symbol - cuGetProcAddress_fnptr = &cuGetProcAddress; -#endif - + get_symbol(STRINGIFY(cuGetProcAddress), cuGetProcAddress_fnptr); if(cuGetProcAddress_fnptr != nullptr) { -#define DRIVER_GET_FNPTR(name, ver) \ +#define GET_SYMBOL(name, ver) \ cuGetProcAddress_stable(cuGetProcAddress_fnptr, name##_fnptr, #name, ver, \ "Could not retrieve symbol " #name); - - CUDA_DRIVER_APIS(DRIVER_GET_FNPTR); -#undef DRIVER_GET_FNPTR + CUDA_DRIVER_APIS(GET_SYMBOL); +#undef GET_SYMBOL } else { -#if defined(REALM_USE_LIBDL) -#define DRIVER_GET_FNPTR(name, ver) \ - if(CUDA_SUCCESS != (nullptr != (name##_fnptr = reinterpret_cast( \ - dlsym(libcuda, STRINGIFY(name)))))) { \ - log_gpu.info() << "Could not retrieve symbol " #name; \ +#define GET_SYMBOL(name, ver) \ + if(!get_symbol(STRINGIFY(name), name##_fnptr)) { \ + log_gpu.info("Could not retrieve symbol %s", STRINGIFY(name)); \ } - CUDA_DRIVER_APIS(DRIVER_GET_FNPTR) -#undef DRIVER_GET_FNPTR -#else -#define DRIVER_GET_FNPTR(name, ver) name##_fnptr = &name; - // Only enumerate the driver apis for the base toolkit version, extra features - // cannot be enumerated - CUDA_DRIVER_APIS_BASE(DRIVER_GET_FNPTR); -#undef DRIVER_GET_FNPTR -#endif /* REALM_USE_LIBDL */ + CUDA_DRIVER_APIS(GET_SYMBOL); +#undef GET_SYMBOL } + return true; + } - cuda_api_fnptrs_loaded = true; + bool NVMLLoader::load_symbols() + { +#define GET_SYMBOL(name) \ + if(!get_symbol(STRINGIFY(name), name##_fnptr)) { \ + log_gpu.info("Could not retrieve symbol %s", STRINGIFY(name)); \ + } + NVML_APIS(GET_SYMBOL); + return true; + } + bool CUPTILoader::load_symbols() + { +#define GET_SYMBOL(name) \ + if(!get_symbol(STRINGIFY(name), name##_fnptr)) { \ + log_gpu.info("Could not retrieve symbol %s", STRINGIFY(name)); \ + } + CUPTI_APIS(GET_SYMBOL); return true; } - static bool resolve_nvml_api_fnptrs() + static bool resolve_cuda_api_fnptrs(void) { -#ifdef REALM_USE_LIBDL - void *libnvml = NULL; - if(nvml_api_fnptrs_loaded) +#if !defined(REALM_CUDA_DYNAMIC_LOAD) + return true; +#else + if(cuda_loader) { return true; - log_gpu.info() << "dynamically loading libnvidia-ml.so"; - libnvml = dlopen("libnvidia-ml.so.1", RTLD_NOW); - if(libnvml == NULL) { - log_gpu.info() << "could not open libnvidia-ml.so" << strerror(errno); - return false; } -#define DRIVER_GET_FNPTR(name) \ - do { \ - void *sym = dlsym(libnvml, STRINGIFY(name)); \ - if(!sym) { \ - log_gpu.info() << "symbol '" STRINGIFY(name) " missing from libnvidia-ml.so!"; \ - } \ - name##_fnptr = reinterpret_cast(sym); \ - } while(0) - - NVML_APIS(DRIVER_GET_FNPTR); -#undef DRIVER_GET_FNPTR - - nvml_api_fnptrs_loaded = true; - return true; + return cuda_loader.load({ +#if defined(REALM_ON_WINDOWS) + "nvcuda.dll", #else - return false; + "libcuda.so.1", +#endif + }); #endif } - static bool resolve_cupti_api_fnptrs() + static bool resolve_nvml_api_fnptrs() { -#if defined(REALM_USE_LIBDL) - void *libcupti = NULL; - if(cupti_api_fnptrs_loaded) { + if(nvml_loader) { return true; } - log_gpu.info("dynamically loading libcupti.so"); - libcupti = dlopen("libcupti.so", RTLD_NOW); - if(libcupti == NULL) { - log_gpu.info("Failed to retrieve libcupti.so from LD_LIBRARY_PATH, trying " - "/usr/local/cuda/extras/CUPTI/lib64!"); - libcupti = dlopen("/usr/local/cuda/extras/CUPTI/lib64/libcupti.so", RTLD_NOW); - if(libcupti == NULL) { - log_gpu.info() << "Could not open libcupti.so" << strerror(errno); - return false; - } - } - -#define DRIVER_GET_FNPTR(name) \ - do { \ - void *sym = dlsym(libcupti, STRINGIFY(name)); \ - if(!sym) { \ - log_gpu.info() << "symbol '" STRINGIFY(name) " missing from libcupti.so!"; \ - } \ - name##_fnptr = reinterpret_cast(sym); \ - } while(0) - - CUPTI_APIS(DRIVER_GET_FNPTR); -#undef DRIVER_GET_FNPTR - - log_gpu.info() << "Loaded cupti!"; - cupti_api_fnptrs_loaded = true; - return true; + return nvml_loader.load({ +#if defined(REALM_ON_WINDOWS) + "nvml.dll", #else - return false; + "libnvidia-ml.so", #endif + }); + } + + static bool resolve_cupti_api_fnptrs() + { + if(cupti_loader) { + return true; + } + return cupti_loader.load( + {"libcupti.so", "/usr/local/cuda/extras/CUPTI/libcupti.so"}); } /*static*/ ModuleConfig *CudaModule::create_module_config(RuntimeImpl *runtime) diff --git a/src/realm/gasnetex/gasnetex_internal.cc b/src/realm/gasnetex/gasnetex_internal.cc index 7060d3e4ec..38d2cc1b8c 100644 --- a/src/realm/gasnetex/gasnetex_internal.cc +++ b/src/realm/gasnetex/gasnetex_internal.cc @@ -22,6 +22,7 @@ #include "realm/runtime_impl.h" #include "realm/mem_impl.h" #include "realm/logging.h" +#include "realm/loader.h" #ifdef REALM_USE_CUDA #include "realm/cuda/cuda_module.h" @@ -33,8 +34,6 @@ #include "realm/hip/hip_internal.h" #endif -#include - namespace Realm { // defined in gasnetex_module.cc @@ -3178,25 +3177,24 @@ namespace Realm { static const char default_gex_wrapper_name[] = "librealm_gex_wrapper.so"; const char *gex_wrapper_name = getenv("REALM_GASNETEX_WRAPPER"); gex_wrapper_init_pfn realm_gex_wrapper_init_fnptr = nullptr; - void *librealm_gex_wrapper_handle = nullptr; + lib_handle_t librealm_gex_wrapper_handle = nullptr; if(gex_wrapper_name == nullptr) { gex_wrapper_name = default_gex_wrapper_name; } log_gex.debug("Loading gex wrapper: %s", gex_wrapper_name); - librealm_gex_wrapper_handle = dlopen(gex_wrapper_name, RTLD_NOW); + librealm_gex_wrapper_handle = Realm::load_library(gex_wrapper_name, LOADLIB_NOW); if(librealm_gex_wrapper_handle == nullptr) { log_gex.error("Failed to load gex wrapper at %s", gex_wrapper_name); goto Error; } realm_gex_wrapper_init_fnptr = reinterpret_cast( - dlsym(librealm_gex_wrapper_handle, "realm_gex_wrapper_init")); + Realm::get_symbol(librealm_gex_wrapper_handle, "realm_gex_wrapper_init")); if(realm_gex_wrapper_init_fnptr == nullptr) { - const char *dlsym_error = dlerror(); - log_gex.error("Cannot load wrapper entry symbol: %s\n", dlsym_error); - dlclose(librealm_gex_wrapper_handle); + log_gex.error("Cannot load wrapper entry symbol\n"); + Realm::close_library(librealm_gex_wrapper_handle); goto Error; } if(0 != realm_gex_wrapper_init_fnptr(&gex_wrapper_handle)) { diff --git a/src/realm/loader.cc b/src/realm/loader.cc new file mode 100644 index 0000000000..b93be07d73 --- /dev/null +++ b/src/realm/loader.cc @@ -0,0 +1,78 @@ +/* + * Copyright 2025 Stanford University, NVIDIA Corporation + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#if defined(REALM_ON_WINDOWS) +#define WIN32_LEAN_AND_MEAN 1 +#include +#else +#include +#endif + +namespace Realm { + + lib_handle_t load_library(const char *name, int flags) + { +#if defined(REALM_ON_WINDOWS) + return static_cast(LoadLibrary(name)); +#elif defined(REALM_USE_DLFCN) + int dlopen_flags = 0; + if(flags & LOADLIB_NOW) { + dlopen_flags |= RTLD_NOW; + } else { + dlopen_flags |= RTLD_LAZY; + } + lib_handle_t handle = dlopen(name, dlopen_flags); + if(handle == nullptr) { + dlerror(); // Clear error + } + return handle; +#else + return nullptr; +#endif + } + + void close_library(lib_handle_t hdl) + { +#if defined(REALM_ON_WINDOWS) + FreeLibrary(static_cast(hdl)); +#elif defined(REALM_USE_DLFCN) + dlclose(hdl); +#endif + } + + void *get_symbol(lib_handle_t hdl, const char *name) + { +#if defined(REALM_ON_WINDOWS) + if(hdl == THIS_LIB) { + hdl = reinterpret_cast(GetModuleHandle(nullptr)); + } + return GetProcAddress(static_cast(hdl), name); +#elif defined(REALM_USE_DLFCN) + void *sym = dlsym(hdl, name); + if(sym == nullptr) { + dlerror(); + } + return sym; +#else + return nullptr; +#endif + } + +} // namespace Realm \ No newline at end of file diff --git a/src/realm/loader.h b/src/realm/loader.h new file mode 100644 index 0000000000..161dc23d29 --- /dev/null +++ b/src/realm/loader.h @@ -0,0 +1,117 @@ +/* + * Copyright 2025 Stanford University, NVIDIA Corporation + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __LOADER_H__ +#define __LOADER_H__ + +#include +#include +#include +namespace Realm { + + typedef void *lib_handle_t; + + static constexpr lib_handle_t THIS_LIB = nullptr; + + enum LoadLibraryFlags + { + LOADLIB_NOW = 1, + LOADLIB_DEFAULT = LOADLIB_NOW, + }; + + /// @brief Helper to load a library from the given name + /// @param name Name of the library to load + /// @param flags Extra flags for how the library should be loaded + /// @return handle for the given library + lib_handle_t load_library(const char *name, int flags = LOADLIB_DEFAULT); + + /// @brief Helper to close / free a library handle from \sa load_library + /// @param hdl Handle to release + void close_library(lib_handle_t hdl); + + /// @brief Helper to retrieve a symbol from a loaded library by it's symbol name + /// @param hdl Handle to retrieve from + /// @param name Name of the symbol to retrieve + /// @return Address of the symbol, or nullptr if not found. + void *get_symbol(lib_handle_t hdl, const char *name); + + /// @brief Class to hold a reference to a dynamically loaded library + /// @tparam SymbolTable The class whose load_symbols() function will be called to + /// initialize all it's symbols + template + class Loader { + lib_handle_t handle = nullptr; + + public: + Loader() = default; + Loader(lib_handle_t &hdl) + : handle(std::move(hdl)) + { + hdl = nullptr; + } + Loader(const Loader &) = delete; + Loader(Loader &&) = delete; + Loader &operator=(Loader &&) = delete; + Loader &operator=(Loader &) = delete; + + ~Loader() + { + if(handle != nullptr) { + Realm::close_library(handle); + } + } + operator bool() const { return handle != nullptr; } + + /// @brief Initializes the loader from the given name + bool load(const char *name, int flags = LOADLIB_DEFAULT) + { + handle = Realm::load_library(name, flags); + if(handle == nullptr) { + return false; + } + if(!static_cast(this)->load_symbols()) { + Realm::close_library(handle); + handle = nullptr; + } + return handle != nullptr; + } + + /// @brief Initializes the loader from the given names + bool load(std::initializer_list names, int flags = LOADLIB_DEFAULT) + { + for(const char *name : names) { + if(load(name, flags)) { + return true; + } + } + return false; + } + + /// @brief Gets the address of the symbol for the given symbol name + /// @tparam T type of the object this symbol references (must be a pointer type) + /// @param name Name of the symbol this references + /// @return True if successful, false otherwise. + template ::value>> + bool get_symbol(const char *name, T &ptr) + { + ptr = reinterpret_cast(Realm::get_symbol(handle, name)); + return ptr != nullptr; + } + }; + +} // namespace Realm +#endif // __LOADER_H__ \ No newline at end of file diff --git a/src/realm/module.cc b/src/realm/module.cc index 81c0747b0b..00d8ed13aa 100644 --- a/src/realm/module.cc +++ b/src/realm/module.cc @@ -18,19 +18,14 @@ // Realm modules #include "realm/realm_config.h" - #include "realm/module.h" - #include "realm/logging.h" +#include "realm/loader.h" #include #include #include -#ifdef REALM_USE_DLFCN -#include -#endif - // TODO: replace this with Makefile (or maybe cmake) magic that adapts automatically // to the build-system-controlled list of statically-linked Realm modules @@ -215,8 +210,7 @@ namespace Realm { /*static*/ bool ModuleRegistrar::check_symbol_visibility(void) { #ifdef REALM_USE_DLFCN - void *sym = dlsym(RTLD_DEFAULT, "realm_internal_api_symbols_visible"); - (void)dlerror(); // clear any lookup error + void *sym = Realm::get_symbol(Realm::THIS_LIB, "realm_internal_api_symbols_visible"); return (sym == &realm_internal_api_symbols_visible); #else // definitely won't work without dlfcn @@ -228,7 +222,7 @@ namespace Realm { // accepts a colon-separated list of so files to try to load static void load_module_list(const char *sonames, RuntimeImpl *runtime, std::vector &cmdline, - std::vector &handles) + std::vector &handles) { // null/empty strings are nops if(!sonames || !*sonames) @@ -253,22 +247,19 @@ namespace Realm { // skip the color after the filename (if it exists) p1 = p2 + (*p2 ? 1 : 0); - // no leftover errors from anybody else please... - assert(dlerror() == 0); - // open so file, resolving all symbols but not polluting global namespace - void *handle = dlopen(filename, RTLD_NOW | RTLD_LOCAL); + lib_handle_t handle = Realm::load_library(filename, LOADLIB_NOW); if(handle == 0) { - log_module.error() << "could not load " << filename << ": " << dlerror(); + log_module.error() << "could not load " << filename; continue; } { // this file should have a "realm_module_version" symbol - void *sym = dlsym(handle, "realm_module_version"); + void *sym = Realm::get_symbol(handle, "realm_module_version"); if(!sym) { log_module.error() << "symbol 'realm_module_version' not found in " << filename; - dlclose(handle); + Realm::close_library(handle); continue; } const char *module_version = static_cast(sym); @@ -285,7 +276,7 @@ namespace Realm { << "module version mismatch in '" << filename << "': realm='" << REALM_VERSION << "' module='" << module_version << "' - set REALM_PERMIT_MODULE_VERSION_MISMATCH to load anyway"; - dlclose(handle); + Realm::close_library(handle); continue; } } @@ -331,9 +322,9 @@ namespace Realm { abort(); } - for(std::vector::const_iterator it = sonames_list.begin(); - it != sonames_list.end(); it++) - load_module_list(it->c_str(), runtime, cmdline, module_sofile_handles); + for(std::string &name : sonames_list) { + load_module_list(name.c_str(), runtime, cmdline, module_sofile_handles); + } sofile_loaded = true; #else log_module.fatal() @@ -357,10 +348,8 @@ namespace Realm { if(module_sofile_handles.size() > 0) { assert(sofile_loaded); } - for(std::vector::iterator it = module_sofile_handles.begin(); - it != module_sofile_handles.end(); it++) { - void *handle = *it; - void *sym = dlsym(handle, "create_realm_module"); + for(lib_handle_t handle : module_sofile_handles) { + void *sym = Realm::get_symbol(handle, "create_realm_module"); if(!sym) { log_module.error() << "symbol 'create_realm_module' not found"; continue; @@ -379,11 +368,9 @@ namespace Realm { { load_module_sofiles(cmdline); #ifdef REALM_USE_DLFCN - for(std::vector::iterator it = module_sofile_handles.begin(); - it != module_sofile_handles.end(); it++) { - void *handle = *it; - void *sym = dlsym(handle, "create_realm_module_config"); - if(!sym) { + for(lib_handle_t handle : module_sofile_handles) { + void *sym = Realm::get_symbol(handle, "create_realm_module_config"); + if(sym == nullptr) { log_module.error() << "symbol 'create_realm_module_config' not found"; continue; } @@ -398,14 +385,9 @@ namespace Realm { { #ifdef REALM_USE_DLFCN while(!sofile_handles.empty()) { - void *handle = sofile_handles.back(); + lib_handle_t handle = sofile_handles.back(); sofile_handles.pop_back(); - -#ifndef NDEBUG - int ret = -#endif - dlclose(handle); - assert(ret == 0); + Realm::close_library(handle); } #endif } diff --git a/src/realm/module.h b/src/realm/module.h index 7131a479ca..770381ac8a 100644 --- a/src/realm/module.h +++ b/src/realm/module.h @@ -22,6 +22,7 @@ #include "realm/realm_config.h" #include "realm/module_config.h" +#include "realm/loader.h" // to provide Realm functionality via the module interface, you need to: // @@ -201,8 +202,8 @@ namespace Realm { protected: RuntimeImpl *runtime; bool sofile_loaded; - std::vector module_sofile_handles; - std::vector network_sofile_handles; + std::vector module_sofile_handles; + std::vector network_sofile_handles; }; // macros used within a module when being built as a dynamic shared object diff --git a/src/realm/network.cc b/src/realm/network.cc index 345a63feee..4ad800d3b5 100644 --- a/src/realm/network.cc +++ b/src/realm/network.cc @@ -21,10 +21,7 @@ #include "realm/cmdline.h" #include "realm/logging.h" #include "realm/activemsg.h" - -#ifdef REALM_USE_DLFCN -#include -#endif +#include "realm/loader.h" static void *aligned_malloc(size_t bytes, size_t alignment) { @@ -532,7 +529,7 @@ namespace Realm { // accepts a colon-separated list of so files to try to load static int load_network_module_list(const char *sonames, RuntimeImpl *runtime, int *argc, const char ***argv, - std::vector &handles, + std::vector &handles, std::vector &modules) { // null/empty strings are nops @@ -559,23 +556,20 @@ namespace Realm { // skip the color after the filename (if it exists) p1 = p2 + (*p2 ? 1 : 0); - // no leftover errors from anybody else please... - assert(dlerror() == 0); - // open so file, resolving all symbols but not polluting global namespace - void *handle = dlopen(filename, RTLD_NOW | RTLD_LOCAL); - if(handle == 0) { - std::cerr << "ERROR: could not load " << filename << ": " << dlerror() << "\n"; + lib_handle_t handle = Realm::load_library(filename, LOADLIB_NOW); + if(handle == nullptr) { + std::cerr << "ERROR: could not load " << filename << "\n"; continue; } { // this file should have a "realm_module_version" symbol - void *sym = dlsym(handle, "realm_module_version"); + void *sym = Realm::get_symbol(handle, "realm_module_version"); if(!sym) { std::cerr << "ERROR: symbol 'realm_module_version' not found in '" << filename << "'\n"; - dlclose(handle); + Realm::close_library(handle); continue; } const char *module_version = static_cast(sym); @@ -591,18 +585,18 @@ namespace Realm { std::cerr << "ERROR: module version mismatch in '" << filename << "': realm='" << REALM_VERSION << "' module='" << module_version << "' - set REALM_PERMIT_MODULE_VERSION_MISMATCH to load anyway\n"; - dlclose(handle); + Realm::close_library(handle); continue; } } } // this file should also have a "create_realm_network_module" symbol - void *sym = dlsym(handle, "create_realm_network_module"); + void *sym = Realm::get_symbol(handle, "create_realm_network_module"); if(!sym) { std::cerr << "ERROR: symbol 'create_realm_network_module' not found in '" << filename << "'\n"; - dlclose(handle); + Realm::close_library(handle); continue; } diff --git a/src/realm/ucx/bootstrap/bootstrap_loader.cc b/src/realm/ucx/bootstrap/bootstrap_loader.cc index 12f5e4198e..bf308facb7 100644 --- a/src/realm/ucx/bootstrap/bootstrap_loader.cc +++ b/src/realm/ucx/bootstrap/bootstrap_loader.cc @@ -23,15 +23,13 @@ #include "realm/realm_config.h" #include "realm/ucx/bootstrap/bootstrap_util.h" #include "realm/ucx/bootstrap/bootstrap_internal.h" +#include "realm/loader.h" #define GET_SYMBOL(lib_handle, name, var, status) \ do { \ - void **var_ptr = (void **)&(var); \ - void *tmp = (void *)dlsym(lib_handle, name); \ - BOOTSTRAP_NULL_ERROR_JMP(tmp, status, BOOTSTRAP_ERROR_INTERNAL, out, \ - "Bootstrap failed to get symbol '%s'\n\t%s\n", name, \ - dlerror()); \ - *var_ptr = tmp; \ + var = reinterpret_cast(Realm::get_symbol(lib_handle, name)); \ + BOOTSTRAP_NULL_ERROR_JMP(var, status, BOOTSTRAP_ERROR_INTERNAL, out, \ + "Bootstrap failed to get symbol '%s'\n\t%s\n", name); \ } while(0) static void *plugin_hdl; @@ -58,14 +56,11 @@ namespace Realm { int (*bootstrap_plugin_init)(void *arg, bootstrap_handle_t *handle); int status = 0; - dlerror(); /* Clear any existing error */ plugin_name = strdup(plugin); plugin_hdl = dlopen(plugin, RTLD_NOW); BOOTSTRAP_NULL_ERROR_JMP(plugin_hdl, status, -1, error, - "Bootstrap unable to load '%s'\n\t%s\n", plugin, - dlerror()); + "Bootstrap unable to load '%s'\n", plugin); - dlerror(); /* Clear any existing error */ GET_SYMBOL(plugin_hdl, "realm_ucp_bootstrap_plugin_init", bootstrap_plugin_init, status); diff --git a/src/realm/ucx/ucp_internal.cc b/src/realm/ucx/ucp_internal.cc index ce5839b747..168f6d22cd 100644 --- a/src/realm/ucx/ucp_internal.cc +++ b/src/realm/ucx/ucp_internal.cc @@ -888,7 +888,7 @@ namespace Realm { { log_ucp.info() << "dynamically loading libucp.so.0"; - libucp = dlopen("libucp.so.0", RTLD_NOW); + libucp = Realm::load_library("libucp.so.0"); if(!libucp) { log_ucp.warning() << "could not open libucp.so.0: " << strerror(errno); return false; @@ -897,7 +897,7 @@ namespace Realm { #define STRINGIFY(s) #s #define UCP_GET_FNPTR(name) \ do { \ - void *sym = dlsym(libucp, STRINGIFY(name)); \ + void *sym = Realm::get_symbol(libucp, STRINGIFY(name)); \ if(!sym) { \ log_ucp.warning() << "symbol '" STRINGIFY(name) "' missing from libucp.so!"; \ return false; \ @@ -943,7 +943,7 @@ namespace Realm { err_version: #ifdef REALM_UCX_DYNAMIC_LOAD - dlclose(libucp); + Realm::close_library(libucp); err: #endif return false; diff --git a/src/realm/ucx/ucp_internal.h b/src/realm/ucx/ucp_internal.h index d16f800f3f..7d49fcee49 100644 --- a/src/realm/ucx/ucp_internal.h +++ b/src/realm/ucx/ucp_internal.h @@ -36,6 +36,7 @@ #include "realm/ucx/spinlock.h" #include "realm/ucx/bootstrap/bootstrap_internal.h" #include "realm/ucx/ucc_comm.h" +#include "realm/loader.h" #include #include @@ -250,7 +251,7 @@ namespace Realm { using AttachMap = std::unordered_map>; #ifdef REALM_UCX_DYNAMIC_LOAD - void *libucp{nullptr}; + libhandle_t libucp{nullptr}; #endif bool initialized_boot{false}; bool initialized_ucp{false}; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 77bc2d6efb..a4bebcec00 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -90,6 +90,12 @@ if(googletest_ADDED) set_property(GLOBAL PROPERTY PACKAGES_NOT_FOUND "${_NOT_FOUND}") endif() +add_library(dummy MODULE EXCLUDE_FROM_ALL ${REALM_TEST_DIR}/unit_tests/dummy_library.cc) +generate_export_header( + dummy EXPORT_FILE_NAME "${CMAKE_CURRENT_BINARY_DIR}/dummy_library_exports.h" +) +target_include_directories(dummy PRIVATE "${CMAKE_CURRENT_BINARY_DIR}") + list( APPEND REALM_UNIT_TESTS @@ -121,15 +127,19 @@ list( hpp_span_test.cc hpp_runtime_test.cc hpp_event_test.cc + loader_test.cc ) list(TRANSFORM REALM_UNIT_TESTS PREPEND "${REALM_TEST_DIR}/unit_tests/") add_executable(realm_unit_tests ${REALM_UNIT_TESTS}) +# Add the dummy library as a dependency for building +add_dependencies(realm_unit_tests dummy) # Needed for designated initializers # TODO(apriakhin): to be removed target_compile_features(realm_unit_tests PUBLIC cxx_std_20) target_link_libraries( realm_unit_tests PRIVATE realm_obj GTest::gmock_main ${REALM_LIBRARIES} ${REALM_EXTRA_OBJS} ) +target_include_directories(realm_unit_tests PRIVATE "${CMAKE_CURRENT_BINARY_DIR}") # TODO(wei): To be merged into realm_unit_tests once runtime singleton is # removed from all unit tests @@ -166,7 +176,8 @@ if((NOT REALM_SANITIZER OR REALM_SANITIZER STREQUAL "NONE") gtest_discover_tests(realm_unit_tests NO_PRETTY_TYPES NO_PRETTY_VALUES PROPERTIES LABELS "unit") gtest_discover_tests(realm_c_unit_tests NO_PRETTY_TYPES NO_PRETTY_VALUES PROPERTIES LABELS "unit") else() - add_test(NAME realm_unit_tests COMMAND $) + add_test(NAME realm_unit_tests COMMAND $ + WORKING_DIRECTORY $) add_test(NAME realm_c_unit_tests COMMAND $) set_tests_properties(realm_unit_tests realm_c_unit_tests PROPERTIES LABELS "unit") endif() diff --git a/tests/unit_tests/dummy_library.cc b/tests/unit_tests/dummy_library.cc new file mode 100644 index 0000000000..ec2cf2678a --- /dev/null +++ b/tests/unit_tests/dummy_library.cc @@ -0,0 +1,20 @@ +/* + * Copyright 2025 Stanford University, NVIDIA Corporation + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "dummy_library.h" + +extern "C" int dummy(int x) { return x; } \ No newline at end of file diff --git a/tests/unit_tests/dummy_library.h b/tests/unit_tests/dummy_library.h new file mode 100644 index 0000000000..7b07389670 --- /dev/null +++ b/tests/unit_tests/dummy_library.h @@ -0,0 +1,33 @@ +/* + * Copyright 2025 Stanford University, NVIDIA Corporation + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __DUMMY_LIB_H__ +#define __DUMMY_LIB_H__ + +#include "dummy_library_exports.h" + +#if defined(__cplusplus) +extern "C" { +#endif // __cplusplus + +int DUMMY_EXPORT dummy(int x); + +#if defined(__cplusplus) +} +#endif // __cplusplus + +#endif // __DUMMY_LIB_H__ \ No newline at end of file diff --git a/tests/unit_tests/loader_test.cc b/tests/unit_tests/loader_test.cc new file mode 100644 index 0000000000..13e6d9e914 --- /dev/null +++ b/tests/unit_tests/loader_test.cc @@ -0,0 +1,113 @@ +/* + * Copyright 2025 Stanford University, NVIDIA Corporation + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "realm/realm_config.h" +#include "realm/loader.h" +#include "dummy_library.h" +#include + +using namespace Realm; + +#if defined(REALM_ON_WINDOWS) +#define DUMMY_LIB_NAME "./dummy.dll" +#else +#define DUMMY_LIB_NAME "./libdummy.so" +#endif + +struct DummyLoader : public Loader { + decltype(&dummy) dummy_fnptr = nullptr; + bool load_symbols() { return this->get_symbol("dummy", dummy_fnptr); } +}; + +struct DummyBadLoader : public Loader { + decltype(&dummy) dummy_fnptr = nullptr; + bool load_symbols() { return this->get_symbol("dummy.does.not.exist", dummy_fnptr); } +}; + +TEST(Loader, CanLoadDummy) +{ + lib_handle_t dummy_lib = load_library(DUMMY_LIB_NAME); + + EXPECT_NE(dummy_lib, nullptr); + + close_library(dummy_lib); +} + +TEST(Loader, LibraryNotFound) +{ + lib_handle_t dummy_lib = load_library(DUMMY_LIB_NAME ".does_not_exist"); + EXPECT_EQ(dummy_lib, nullptr); +} + +TEST(Loader, CanRetrieveValidSymbol) +{ + const int TEST_VALUE = 0xDEADBEEF; + int ret_value = 0; + lib_handle_t dummy_lib = load_library(DUMMY_LIB_NAME); + + decltype(&dummy) dummy_fnptr = + reinterpret_cast(get_symbol(dummy_lib, "dummy")); + ASSERT_NE(dummy_fnptr, nullptr); + + ret_value = dummy_fnptr(TEST_VALUE); + close_library(dummy_lib); + + EXPECT_EQ(TEST_VALUE, ret_value); +} + +TEST(Loader, SymbolNotFound) +{ + lib_handle_t dummy_lib = load_library(DUMMY_LIB_NAME); + void *fnptr = get_symbol(dummy_lib, "bar_does_not_exist"); + close_library(dummy_lib); + + EXPECT_EQ(fnptr, nullptr); +} + +TEST(Loader, DefaultConstructedInvalid) +{ + DummyLoader loader; + EXPECT_FALSE(!!loader); +} + +TEST(Loader, LoaderClassLoads) +{ + const int TEST_VALUE = 0xCAFEBABE; + DummyLoader loader; + EXPECT_TRUE(loader.load(DUMMY_LIB_NAME)); + EXPECT_TRUE(!!loader); + EXPECT_NE(loader.dummy_fnptr, nullptr); + EXPECT_EQ(TEST_VALUE, loader.dummy_fnptr(TEST_VALUE)); +} + +TEST(Loader, LoaderClassLoadMultipleNames) +{ + const int TEST_VALUE = 0xCAFEBABE; + DummyLoader loader; + EXPECT_TRUE(loader.load({DUMMY_LIB_NAME ".does_not_exist", DUMMY_LIB_NAME})); + EXPECT_TRUE(!!loader); + EXPECT_NE(loader.dummy_fnptr, nullptr); + EXPECT_EQ(TEST_VALUE, loader.dummy_fnptr(TEST_VALUE)); +} + +TEST(Loader, LoaderClassSymbolFails) +{ + const int TEST_VALUE = 0xCAFEBABE; + DummyBadLoader loader; + EXPECT_FALSE(loader.load(DUMMY_LIB_NAME)); + EXPECT_FALSE(!!loader); +} \ No newline at end of file