diff --git a/CMakeLists.txt b/CMakeLists.txt index e3aeefac7..932ee0111 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -629,7 +629,7 @@ if(OCOS_ENABLE_VISION) if (NOT OCOS_ENABLE_DLIB) message(FATAL_ERROR "Vision operators require DLIB to be enabled.") # for now, we need dlib for image processing endif() - include(ext_imgcodecs) + include(ext_imgcodecs) target_include_directories(ocos_operators PUBLIC ${libPNG_SOURCE_DIR} ${libJPEG_SOURCE_DIR}) target_link_libraries(ocos_operators PUBLIC ${PNG_LIBRARY} ${JPEG_LIBRARY}) endif() @@ -763,9 +763,14 @@ if(OCOS_BUILD_SHARED_LIB) source_group(TREE ${PROJECT_SOURCE_DIR} FILES ${shared_TARGET_SRC}) standardize_output_folder(extensions_shared) - if(LINUX OR ANDROID) - set_property(TARGET extensions_shared APPEND_STRING PROPERTY LINK_FLAGS - " -Wl,--version-script -Wl,${PROJECT_SOURCE_DIR}/shared/ortcustomops.ver") + if(LINUX OR ANDROID OR CMAKE_SYSTEM_NAME STREQUAL "AIX") + if(CMAKE_SYSTEM_NAME STREQUAL "AIX") + set_property(TARGET extensions_shared APPEND_STRING PROPERTY LINK_FLAGS + " -Wl,-bE:${PROJECT_SOURCE_DIR}/shared/ortcustomops.exp") + else() + set_property(TARGET extensions_shared APPEND_STRING PROPERTY LINK_FLAGS + " -Wl,--version-script -Wl,${PROJECT_SOURCE_DIR}/shared/ortcustomops.ver") + endif() # strip if not a debug build if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") set_property(TARGET extensions_shared APPEND_STRING PROPERTY LINK_FLAGS " -Wl,-s") diff --git a/cmake/externals/blingfire-aix.patch b/cmake/externals/blingfire-aix.patch new file mode 100644 index 000000000..699fe4554 --- /dev/null +++ b/cmake/externals/blingfire-aix.patch @@ -0,0 +1,42 @@ +diff --git a/blingfireclient.library/inc/FAConfig.h b/blingfireclient.library/inc/FAConfig.h +index 8dfa36e..b6c7cb1 100644 +--- a/blingfireclient.library/inc/FAConfig.h ++++ b/blingfireclient.library/inc/FAConfig.h +@@ -23,7 +23,7 @@ + #include + #include + +-#ifdef BLING_FIRE_MAC ++#if defined(BLING_FIRE_MAC) || defined(_AIX) + #include + #else + #include + + + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 5d47b3a..428c888 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -26,7 +26,11 @@ IF (WIN32 AND MSVC) + add_link_options("$<$>:/LTCG>") + set (default_build_type "RelWithDebInfo") + ELSE() +- set (CMAKE_CXX_FLAGS " -std=c++11 -fpic ${OPTIMIZE} -DNDEBUG") ++ IF(AIX) ++ set (CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -std=c++11 -fpic ${OPTIMIZE} -DNDEBUG") ++ ELSE() ++ set (CMAKE_CXX_FLAGS " -std=c++11 -fpic ${OPTIMIZE} -DNDEBUG") ++ ENDIF() + IF(APPLE) + set (CMAKE_CXX_FLAGS_RELEASE " -std=c++11 -fpic ${OPTIMIZE} -DNDEBUG -ffunction-sections -fdata-sections -Wl,-dead_strip") + ELSE() +@@ -107,7 +111,7 @@ FOREACH(dir ${SUBDIRS}) + IF(${dirname} STREQUAL "any_test") + target_link_libraries(${dirname} ${CMAKE_DL_LIBS}) + ENDIF() +- IF (APPLE) ++ IF (APPLE OR CMAKE_SYSTEM_NAME MATCHES "AIX") + target_link_libraries(${dirname} iconv) + ENDIF() + ENDIF() diff --git a/cmake/externals/blingfire.cmake b/cmake/externals/blingfire.cmake index 8c7eefe1c..c1ba2f24b 100644 --- a/cmake/externals/blingfire.cmake +++ b/cmake/externals/blingfire.cmake @@ -3,7 +3,13 @@ FetchContent_Declare( GIT_REPOSITORY https://github.com/microsoft/BlingFire.git GIT_TAG 0831265c1aca95ca02eca5bf1155e4251e545328 EXCLUDE_FROM_ALL - PATCH_COMMAND git checkout . && git apply --ignore-space-change --ignore-whitespace ${PROJECT_SOURCE_DIR}/cmake/externals/blingfire_cmake.patch) + PATCH_COMMAND git checkout . && git apply --ignore-space-change --ignore-whitespace ${PROJECT_SOURCE_DIR}/cmake/externals/blingfire_cmake.patch && git apply --ignore-space-change --ignore-whitespace ${PROJECT_SOURCE_DIR}/cmake/externals/blingfire-aix.patch + ) + +if(CMAKE_SYSTEM_NAME STREQUAL "AIX") + find_package(Iconv REQUIRED) + link_libraries(Iconv::Iconv) +endif() FetchContent_MakeAvailable(Blingfire) set_target_properties(bingfirtinydll_static PROPERTIES FOLDER diff --git a/cmake/externals/opencv.cmake b/cmake/externals/opencv.cmake index 56b850da0..76ffabe41 100644 --- a/cmake/externals/opencv.cmake +++ b/cmake/externals/opencv.cmake @@ -149,8 +149,8 @@ FetchContent_Declare( -DBUILD_TESTS:BOOL=FALSE -DBUILD_SHARED_LIBS:BOOL=FALSE -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_CURRENT_BINARY_DIR}/opencv - PATCH_COMMAND git checkout . && git apply --whitespace=fix --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/cmake/externals/opencv-no-rtti.patch EXCLUDE_FROM_ALL + PATCH_COMMAND git checkout . && git apply --whitespace=fix --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/cmake/externals/opencv-no-rtti.patch ) FetchContent_MakeAvailable(opencv) diff --git a/cmake/externals/sentencepieceproject.cmake b/cmake/externals/sentencepieceproject.cmake index dfef57654..5f2e8ddf2 100644 --- a/cmake/externals/sentencepieceproject.cmake +++ b/cmake/externals/sentencepieceproject.cmake @@ -39,7 +39,10 @@ endif() # PB files was seperated as another patch file to avoid the patch file too large to be reviewed. set(spm_patches "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject_cmake.patch" - "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject_pb.patch") + "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject_pb.patch" + "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject_aix.patch" + "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject_enum.patch" +) set(spm_patch_command git checkout . && git apply --ignore-space-change --ignore-whitespace ${spm_patches}) diff --git a/cmake/externals/sentencepieceproject_aix.patch b/cmake/externals/sentencepieceproject_aix.patch new file mode 100644 index 000000000..acd7b8bd2 --- /dev/null +++ b/cmake/externals/sentencepieceproject_aix.patch @@ -0,0 +1,15 @@ +diff --git a/src/util.h b/src/util.h +index 0d15863..b2d490f 100644 +--- a/src/util.h ++++ b/src/util.h +@@ -36,7 +36,9 @@ + #include + #endif + +-#if !defined(__APPLE__) && !defined(_WIN32) ++#if defined(_AIX) ++#define IS_BIG_ENDIAN ++#elif !defined(__APPLE__) && !defined(_WIN32) + #include + #if BYTE_ORDER == __BIG_ENDIAN + #define IS_BIG_ENDIAN diff --git a/cmake/externals/sentencepieceproject_enum.patch b/cmake/externals/sentencepieceproject_enum.patch new file mode 100644 index 000000000..778de09cf --- /dev/null +++ b/cmake/externals/sentencepieceproject_enum.patch @@ -0,0 +1,13 @@ +diff --git a/src/unicode_script.h b/src/unicode_script.h +index 917c49d..d249539 100644 +--- a/src/unicode_script.h ++++ b/src/unicode_script.h +@@ -19,7 +19,7 @@ + + namespace sentencepiece { + namespace unicode_script { +-enum ScriptType { ++enum ScriptType:int32_t { + U_Adlam, + U_Ahom, + U_Anatolian_Hieroglyphs, diff --git a/operators/text/vector_to_string.hpp b/operators/text/vector_to_string.hpp index 12e91af06..e13c80602 100644 --- a/operators/text/vector_to_string.hpp +++ b/operators/text/vector_to_string.hpp @@ -13,7 +13,7 @@ namespace std { template struct hash> { - size_t operator()(const vector& __vector) const noexcept; + size_t operator()(const vector& __vec) const noexcept; }; } // namespace std diff --git a/operators/tokenizer/ugm_kernels.hpp b/operators/tokenizer/ugm_kernels.hpp index 2f571a3a5..28bc53a12 100644 --- a/operators/tokenizer/ugm_kernels.hpp +++ b/operators/tokenizer/ugm_kernels.hpp @@ -99,6 +99,12 @@ struct SpmUgmTokenizer { } uint32_t xcda_blob_size; memcpy(&xcda_blob_size, charsmap_data_.data(), sizeof(uint32_t)); + + // Convert from little-endian to host byte order + #if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + xcda_blob_size = __builtin_bswap32(xcda_blob_size); + #endif + charsmap_offset += sizeof(xcda_blob_size); if (xcda_blob_size > charsmap_data_.size() - charsmap_offset) { return OrtxStatus(extError_t::kOrtxErrorCorruptData, "Index out of array bounds in precompiled charsmap!"); @@ -301,7 +307,7 @@ struct SpmUgmTokenizer { "Invalid UTF-8 encoding detected in input string at position " + std::to_string(-validation_result)); } - + std::string normalized; if (case_encoder_) { normalized = NmtNormalize(input); @@ -642,7 +648,12 @@ struct SpmUgmTokenizer { if (index >= xcda_array_size_) { // Fix #2: off-by-one correction ORTX_CXX_API_THROW("[UgmTok]Index out of array bounds in XCDA array!", ORT_RUNTIME_EXCEPTION); } - return xcda_array_[index]; + uint32_t value = xcda_array_[index]; + // Convert from little-endian to host byte order + #if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + value = __builtin_bswap32(value); + #endif + return value; } const uint32_t* xcda_array_; size_t xcda_array_size_; @@ -1067,4 +1078,4 @@ class SpmUgmDecoder { std::set special_token_ids_; }; -} // namespace ort_extensions \ No newline at end of file +} // namespace ort_extensions diff --git a/shared/api/nemo_mel_spectrogram.cc b/shared/api/nemo_mel_spectrogram.cc index 1db5a2087..c05ae6d34 100644 --- a/shared/api/nemo_mel_spectrogram.cc +++ b/shared/api/nemo_mel_spectrogram.cc @@ -18,6 +18,10 @@ #define M_PI 3.14159265358979323846 #endif +#if defined(_AIX) && defined(hz) +#undef hz +#endif + namespace nemo_mel { // Apply pre-emphasis filter: y[n] = x[n] - preemph * x[n-1] diff --git a/shared/api/nemo_mel_spectrogram.h b/shared/api/nemo_mel_spectrogram.h index 7033169ac..c07a9408d 100644 --- a/shared/api/nemo_mel_spectrogram.h +++ b/shared/api/nemo_mel_spectrogram.h @@ -9,6 +9,10 @@ #include #include +#if defined _AIX && defined (hz) +#undef hz +#endif + namespace nemo_mel { struct NemoMelConfig { diff --git a/shared/ortcustomops.exp b/shared/ortcustomops.exp new file mode 100644 index 000000000..e07771273 --- /dev/null +++ b/shared/ortcustomops.exp @@ -0,0 +1,3 @@ +RegisterCustomOps +AddExternalCustomOp +GetActiveOrtAPIVersion