diff --git a/CMakeLists.txt b/CMakeLists.txt index 63e60eb61..0c7d2efd3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,6 +60,8 @@ project(Corrade CXX) # Use folders for nice tree in Visual Studio and XCode set_property(GLOBAL PROPERTY USE_FOLDERS ON) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/modules/" ${CMAKE_MODULE_PATH}) + include(CMakeDependentOption) # Options that used to be unprefixed. New options shouldn't be added to this @@ -187,6 +189,112 @@ if(CORRADE_TARGET_WINDOWS) endif() endif() +# Check if we can use IFUNC for CPU dispatch. Linux with glibc and Android with +# API 18+ has it, but e.g. Alpine Linux with musl doesn't, and on Android with +# API < 30 we don't get AT_HWCAP passed into the resolver and can't call +# getauxval() ourselves because it's too early at that point, which makes it +# pretty useless. Plus it also needs a certain binutils version and a capable +# compiler, so it's easiest to just verify the whole thing. +if(CORRADE_TARGET_UNIX) + include(CheckCXXSourceCompiles) + check_cxx_source_compiles("\ +int fooImplementation() { return 42; } +#if defined(__ANDROID_API__) && __ANDROID_API__ < 30 +#error need Android API 30+ to have AT_HWCAP passed into the resolver +#endif +extern \"C\" int(*fooDispatcher())() { + return fooImplementation; +} +int foo() __attribute__((ifunc(\"fooDispatcher\"))); +int main() { return foo() - 42; }\ + " _CORRADE_CPU_CAN_USE_IFUNC) + # In cases where ifunc is known to be broken, disable it by default -- + # users can still force it to be enabled, but the initial state should not + # cause crashes or other strange behavior. + if(_CORRADE_CPU_CAN_USE_IFUNC) + set(_CORRADE_CPU_USE_IFUNC_DEFAULT ON) + # On GCC 4.8, if --coverage or -fprofile-arcs is enabled, the ifunc + # dispatchers cause a segfault. On Ubuntu 20.04 at least. Not the case + # with GCC 5 there, not the case with GCC 4.8 on Arch. Can't find any + # upstream bug report or commit that would be related to this. + if(CMAKE_CXX_COMPILER_ID AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.9" AND CMAKE_CXX_FLAGS MATCHES "(--coverage|-fprofile-arcs)") + if(NOT DEFINED CORRADE_CPU_USE_IFUNC) + message(WARNING "Disabling CORRADE_CPU_USE_IFUNC by default as it may crash when used together with --coverage on GCC 4.8.") + endif() + set(_CORRADE_CPU_USE_IFUNC_DEFAULT OFF) + endif() + + # If sanitizers are enabled, call into the dispatch function crashes. + # Upstream bugreport https://github.com/google/sanitizers/issues/342 + # suggests using __attribute__((no_sanitize_address)), but that doesn't + # work / can't be used because it would mean marking basically + # everything including the actual implementation that's being + # dispatched to. + # + # Sanitizers can be also enabled any other way such as with + # CMAKE_XCODE_SCHEME_ADDRESS_SANITIZER or with per-target properties + # which isn't easy to detect, so as a secondary measure there's a kill + # switch in Corrade/configure.h that will fail with an error if a + # sanitizer is detected and CORRADE_CPU_USE_IFUNC is enabled. + # + # While Clang and GCC use -fsanitize=whatever, MSVC allows also + # /fsanitize=, so catch both. + if(CMAKE_CXX_FLAGS MATCHES "[-/]fsanitize=") + if(NOT DEFINED CORRADE_CPU_USE_IFUNC) + message(WARNING "Disabling CORRADE_CPU_USE_IFUNC by default as it crashes when used together with sanitizers. See https://github.com/google/sanitizers/issues/342 for more information.") + endif() + set(_CORRADE_CPU_USE_IFUNC_DEFAULT OFF) + endif() + else() + set(_CORRADE_CPU_USE_IFUNC_DEFAULT OFF) + endif() +else() + set(_CORRADE_CPU_CAN_USE_IFUNC OFF) + set(_CORRADE_CPU_USE_IFUNC_DEFAULT OFF) +endif() +cmake_dependent_option(CORRADE_CPU_USE_IFUNC "Allow using GNU IFUNC for runtime CPU dispatch" ${_CORRADE_CPU_USE_IFUNC_DEFAULT} "_CORRADE_CPU_CAN_USE_IFUNC" OFF) + +# Runtime CPU dispatch. Because going through a function pointer may have +# negative perf consequences, enable it by default only on platforms that have +# IFUNC, and thus can avoid the function pointer indirection. +option(CORRADE_BUILD_CPU_RUNTIME_DISPATCH "Build with runtime dispatch for CPU-dependent functionality" ${_CORRADE_CPU_CAN_USE_IFUNC}) +# Enabled by default and independent of CORRADE_BUILD_CPU_RUNTIME_DISPATCH as +# correctness is important. It can however negatively affect performance +# compared to IFUNC or compile-time dispatch, so make it possible to disable +# this for benchmarks. +cmake_dependent_option(CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH "Force pointer-based dispatch for unit tests to verify all variants of CPU-dependent functionality" ON "CORRADE_BUILD_TESTS" OFF) + +# Pass -msimd128 to tests to make it possible to verify also the WASM SIMD +# functionality even if the actual library is built without. I thought it would +# be a no-brainer, however *actual* finalized SIMD128 requires final Clang 13 +# (such version is reported since emsdk 2.0.13, but the actual final version +# containing all intrinsics is only in emsdk 2.0.18), and Node.js 15+, which +# contains V8 8.6: +# https://github.com/nodejs/node/blob/main/doc/changelogs/CHANGELOG_V15.md#2020-10-20-version-1500-current-bethgriggs +# Final bitmask instructions were added in 8.5, Node.js 14 has 8.4: +# https://github.com/v8/v8/commit/aa5bcc09bf7d5d77056e6033918d79c30aa3f49a +# However, even now, in the midst of 2022 wildfires, emsdk still bundles +# Node.js 14.18 and there's no hope of it being upgraded any time soon: +# https://github.com/emscripten-core/emsdk/pull/877 +# https://github.com/emscripten-core/emsdk/issues/947 +# https://github.com/emscripten-core/emsdk/issues/1064 +# So instead of having -msimd128 always, I only enable it implicitly if Node.js +# 15 is found, and then CORRADE_ENABLE_SIMD128 / CORRADE_TARGET_SIMD128 gets +# enabled only if there's Clang 13 but also Emscripten 2.0.18+. What a mess. +if(CORRADE_TARGET_EMSCRIPTEN) + set(ENABLE_IF_NODEJS_15 OFF) + if(CORRADE_BUILD_TESTS) + find_package(NodeJs REQUIRED QUIET) + if(NOT NodeJs_VERSION VERSION_LESS 15.0) + set(ENABLE_IF_NODEJS_15 ON) + elseif(NOT DEFINED CORRADE_BUILD_TESTS_FORCE_WASM_SIMD128) + message(WARNING "Disabling CORRADE_BUILD_TESTS_FORCE_WASM_SIMD128 by default as finalized WebAssembly SIMD support is only since Node.js 15 but found version ${NodeJs_VERSION}") + endif() + endif() + + cmake_dependent_option(CORRADE_BUILD_TESTS_FORCE_WASM_SIMD128 "Force -msimd128 for unit tests to verify also WASM SIMD variants of CPU-dependent functionality" ${ENABLE_IF_NODEJS_15} "CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH" OFF) +endif() + # Backwards compatibility for unprefixed CMake options. If the user isn't # explicitly using prefixed options in the first run already, accept the # unprefixed options, and remember this decision for subsequent runs @@ -256,7 +364,6 @@ if(CORRADE_BUILD_TESTS) enable_testing() endif() -set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/modules/" ${CMAKE_MODULE_PATH}) if(CMAKE_CROSSCOMPILING) find_program(CORRADE_RC_EXECUTABLE corrade-rc) if(NOT CORRADE_RC_EXECUTABLE) diff --git a/doc/Doxyfile b/doc/Doxyfile index 50452ea75..cc77f226b 100644 --- a/doc/Doxyfile +++ b/doc/Doxyfile @@ -2280,14 +2280,14 @@ DOT_NUM_THREADS = 0 # The default value is: Helvetica. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_FONTNAME = Verdana +DOT_FONTNAME = Source Sans Pro # The DOT_FONTSIZE tag can be used to set the size (in points) of the font of # dot graphs. # Minimum value: 4, maximum value: 24, default value: 10. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_FONTSIZE = 8 +DOT_FONTSIZE = 16 # By default doxygen will tell dot to use the default font as specified with # DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set @@ -2445,7 +2445,7 @@ DOT_PATH = # command). # This tag requires that the tag HAVE_DOT is set to YES. -DOTFILE_DIRS = +DOTFILE_DIRS = . # The MSCFILE_DIRS tag can be used to specify one or more directories that # contain msc files that are included in the documentation (see the \mscfile diff --git a/doc/building-corrade.dox b/doc/building-corrade.dox index 94b8cb5a3..136007651 100644 --- a/doc/building-corrade.dox +++ b/doc/building-corrade.dox @@ -550,9 +550,28 @@ Options controlling the build: Corrade features simultaneously in multiple threads. Enabled by default, disable if you don't need this and don't want to pay potential performance penalties coming from thread-local variables. +- `CORRADE_BUILD_CPU_RUNTIME_DISPATCH` --- Build with performance-critical + code paths optimized for multiple architectures (such as SSE or AVX on + x86), with the best matching variant selected at runtime based on detected + CPU features. If not enabled, the library is built with just a single + variant that's picked at compile time depending on target architecture + flags being passed to the compiler. Enabled by default on platforms that + support [GNU IFUNC](https://sourceware.org/glibc/wiki/GNU_IFUNC), which is + currently only Linux with glibc and Android with API 30+. See also the + `CORRADE_CPU_USE_IFUNC` option below and + @ref Cpu-usage-automatic-cached-dispatch for details and information about + performance tradeoffs. Platform-specific options: +- `CORRADE_CPU_USE_IFUNC` --- Allow + [GNU IFUNC](https://sourceware.org/glibc/wiki/GNU_IFUNC) to be used for + runtime dispatch in the @ref Cpu library. Available only on platforms that + support it, which is currently only Linux with glibc and Android with API + 30+, and there enabled by default unless a problematic case is detected + that may cause it to misbehave. See + @ref Cpu-usage-automatic-cached-dispatch for details and information about + performance tradeoffs. - `CORRADE_UTILITY_USE_ANSI_COLORS` --- if building for Windows, this will use ANSI escape codes in @ref Utility::Debug instead of WINAPI functions. Note that you need at least Windows 10 or non-standard console emulator to @@ -622,11 +641,31 @@ compatibility if `CORRADE_BUILD_DEPRECATED` isn't disabled. @subsection building-corrade-tests Building and running tests -If you want to build also the tests (which are not built by default), enable -`CORRADE_BUILD_TESTS` in CMake. The tests use Corrade's own -@ref Corrade::TestSuite "TestSuite" framework and can be run either manually -(the binaries are located in `Test/` subdirectories in the build directory) or -using +Building of tests is controlled by the following options: + +- `CORRADE_BUILD_TESTS` --- Builds unit tests. Disabled by default. +- `CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH` --- Force unit tests to be + built with function-pointer-based dispatch for CPU-dependent functionality, + independently of the `CORRADE_BUILD_CPU_RUNTIME_DISPATCH` and + `CORRADE_CPU_USE_IFUNC` options. This makes it possible for the tests to + verify all variants instead of just one, and is enabled by default. The + overhead from function pointers may however be significant on certain + platforms, skewing benchmark results. Disable it to make the tests use the + same dispatch method as the rest of the library. +- `CORRADE_BUILD_TESTS_FORCE_WASM_SIMD128` --- Force Emscripten unit tests to + be built with `-msimd128` independenently of `CMAKE_CXX_FLAGS`. Together + with `CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH` and presence of the + @ref CORRADE_ENABLE_SIMD128 preprocessor variable this makes it possible + for the tests to verify also WebAssembly SIMD variants of CPU-dependent + functionality instead of just the scalar version if the rest of the library + is built without `-msimd128`. Enabled by default if + `CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH` is set and Node.js version + 15+ is found, which has finalized WebAssembly SIMD support. Disable to make + the tests use the same compilation flags as the rest of the library. + +The tests use Corrade's own @relativeref{Corrade,TestSuite} framework and can +be run either manually (the binaries are located in `Test/` subdirectories in +the build directory) or using @code{.sh} ctest --output-on-failure @@ -947,8 +986,9 @@ for more information. @subsection building-corrade-ci-circleci CircleCI In `package/ci/` there is a `circle.yml` file with Linux GCC 4.8, Linux ARM64, -macOS, Emscripten, AddressSanitizer, ThreadSanitizer, Android x86 and iOS -x86_64 configuration. Online at https://circleci.com/gh/mosra/corrade. +macOS, Emscripten (except SIMD, see the file for details), AddressSanitizer, +ThreadSanitizer, Android x86 and iOS x86_64 configuration. Online at +https://circleci.com/gh/mosra/corrade. @subsection building-corrade-ci-appveyor AppVeyor diff --git a/doc/corrade-changelog.dox b/doc/corrade-changelog.dox index 94e1977f9..8fdc99b31 100644 --- a/doc/corrade-changelog.dox +++ b/doc/corrade-changelog.dox @@ -45,15 +45,18 @@ namespace Corrade { @ref CORRADE_TARGET_POWERPC - New @ref CORRADE_TARGET_SSE3, @ref CORRADE_TARGET_SSSE3, @ref CORRADE_TARGET_SSE41, @ref CORRADE_TARGET_SSE42, - @ref CORRADE_TARGET_AVX, @ref CORRADE_TARGET_AVX_F16C, - @ref CORRADE_TARGET_AVX_FMA, @ref CORRADE_TARGET_AVX2, - @ref CORRADE_TARGET_AVX512F, @ref CORRADE_TARGET_NEON, - @ref CORRADE_TARGET_NEON_FP16, @ref CORRADE_TARGET_NEON_FMA and - @ref CORRADE_TARGET_SIMD128 preprocessor variables added to the already - existing @ref CORRADE_TARGET_SSE2 for detecting enabled instruction sets on - x86, ARM and WebAssembly + @ref CORRADE_TARGET_POPCNT, @ref CORRADE_TARGET_LZCNT, + @ref CORRADE_TARGET_BMI1, @ref CORRADE_TARGET_AVX, + @ref CORRADE_TARGET_AVX_F16C, @ref CORRADE_TARGET_AVX_FMA, + @ref CORRADE_TARGET_AVX2, @ref CORRADE_TARGET_AVX512F, + @ref CORRADE_TARGET_NEON, @ref CORRADE_TARGET_NEON_FMA, + @ref CORRADE_TARGET_NEON_FP16 and @ref CORRADE_TARGET_SIMD128 preprocessor + variables added to the already existing @ref CORRADE_TARGET_SSE2 for + detecting enabled instruction sets on x86, ARM and WebAssembly - New @ref CORRADE_TARGET_32BIT preprocessor variable for cross-platform detection of 32-bit builds +- New @ref Cpu namespace that provides building blocks for compile-time and + runtime CPU feature detection and dispatch on x86, ARM and WebAssembly @subsubsection corrade-changelog-latest-new-containers Containers library @@ -360,6 +363,13 @@ namespace Corrade { Windows as well instead of declaring aligned memory allocation functions on its own, as it's not worth the seriously-looking compiler warnings (see [mosra/corrade#145](https://github.com/mosra/corrade/issues/145)) +- Platforms that support @ref CORRADE_CPU_USE_IFUNC will now build certain + code paths optimized for multiple architectures, with the best variant + selected at runtime using the @ref Cpu library based on available CPU + features. This behavior can be disabled with the + @ref CORRADE_BUILD_CPU_RUNTIME_DISPATCH CMake option. Platforms without + IFUNC support implement runtime dispatch using function pointers instead of + indirect functions and have this currently disabled by default. @subsection corrade-changelog-latest-bugfixes Bug fixes diff --git a/doc/corrade-cmake.dox b/doc/corrade-cmake.dox index ddf3f9c7b..8b87b25ba 100644 --- a/doc/corrade-cmake.dox +++ b/doc/corrade-cmake.dox @@ -211,6 +211,9 @@ also available as preprocessor variables if you include - `CORRADE_BUILD_MULTITHREADED` --- Defined if compiled in a way that makes it possible to safely use certain Corrade features simultaneously in multiple threads. +- `CORRADE_BUILD_CPU_RUNTIME_DISPATCH` --- Defined if built with code paths + optimized for multiple architectres with the best matching variant selected + at runtime based on detected CPU features - `CORRADE_TARGET_UNIX` --- Defined if compiled for some Unix flavor (Linux, BSD, macOS, iOS, Android, ...) - `CORRADE_TARGET_APPLE` --- Defined if compiled for Apple platforms @@ -230,6 +233,9 @@ also available as preprocessor variables if you include - `CORRADE_TARGET_MSVC` --- Defined if compiling with MSVC or Clang with a MSVC frontend - `CORRADE_TARGET_MINGW` --- Defined if compiling under MinGW +- `CORRADE_CPU_USE_IFUNC` - Defined if + [GNU IFUNC](https://sourceware.org/glibc/wiki/GNU_IFUNC) is allowed to be + used for runtime dispatch in the @ref Cpu library - `CORRADE_PLUGINMANAGER_NO_DYNAMIC_PLUGIN_SUPPORT` --- Defined if @ref PluginManager doesn't support dynamic plugin loading due to platform limitations @@ -239,8 +245,11 @@ also available as preprocessor variables if you include used for colored output with @ref Utility::Debug on Windows Besides all the defines above, the @ref Corrade/Corrade.h additionally defines -@ref CORRADE_CXX_STANDARD, @ref CORRADE_TARGET_X86, @ref CORRADE_TARGET_ARM, -@ref CORRADE_TARGET_POWERPC, @ref CORRADE_TARGET_LIBCXX, +@ref CORRADE_CXX_STANDARD, @ref CORRADE_TARGET_X86 and related x86-specific +variables, @ref CORRADE_TARGET_ARM and related ARM-specific variables, +@ref CORRADE_TARGET_POWERPC, @ref CORRADE_TARGET_WASM and related +WebAssembly-specific variables, @ref CORRADE_TARGET_32BIT, +@ref CORRADE_TARGET_BIG_ENDIAN, @ref CORRADE_TARGET_LIBCXX, @ref CORRADE_TARGET_LIBSTDCXX and @ref CORRADE_TARGET_DINKUMWARE based on target architecture, standard and STL used. They are not exposed in CMake because the meaning is unclear with projects that mix more different C++ diff --git a/doc/corrade-developers.dox b/doc/corrade-developers.dox index 214cf9a2a..57e361205 100644 --- a/doc/corrade-developers.dox +++ b/doc/corrade-developers.dox @@ -119,7 +119,7 @@ not strictly required to follow them to the point. - make sure they are mentioned in the library documentation - make sure they are mentioned in building and CMake docs - make sure they are mentioned in `CREDITS.md` - - make sure AppVeyor and Travis downloads them (based on platform + - make sure CircleCI and AppVeyor downloads them (based on platform support) 13. Mention the library in `doc/corrade-changelog.dox` 14. Build documentation: @@ -411,19 +411,20 @@ only in inverse. @section corrade-developers-port Checklist for adding or removing a port -1. Add a new `TARGET_*` variable: +1. Add a new `CORRADE_TARGET_*` CMake variable: - to root `CMakeLists.txt`, which either gets enabled automatically based on system introspection or is exposed through a @cmake option() @ce command - to the list of variables extracted out of `configure.h` in `modules/FindCorrade.cmake` -2. Add a `CORRADE_TARGET_*` variable: - - set it in root `CMakeLists.txt` in case `TARGET_*` is enabled - - add it as a @cpp #cmakedefine @ce macro to `src/Corrade/configure.h.cmake` +2. Add a `CORRADE_TARGET_*` preprocessor variable: + - add it as a @cpp #cmakedefine @ce macro to + `src/Corrade/configure.h.cmake` - add documentation for it to `src/Corrade/Corrade.h` - mention it in `modules/FindCorrade.cmake` docs - mention it in `doc/corrade-cmake.dox` and `doc/building-corrade.dox` -3. Add a new Travis / AppVeyor matrix build for this port (or update existing) +3. Add a new CircleCI / AppVeyor matrix build for this port (or update + existing) 4. Add a new `PKGBUILD-*` file in `package/archlinux` for testing (or update existing) 5. Enable or disable functionality using @cmake if(CORRADE_TARGET_*) @ce in @@ -436,6 +437,135 @@ only in inverse. In order to remove a port, be sure to touch all places mentioned above, only in inverse. +@section corrade-developers-cpu-tag Checklist for adding / removing a CPU instruction set + +1. Add a new `CORRADE_TARGET_FOO` preprocessor variable to `Corrade.h`: + - With relevant documentation pointing ideally to a Wikipedia page + describing the instruction set + - Mentioning the GCC/Clang `-m` option that enables it and the MSVC + equivalent, if any, or the closest `/arch:` option that implies it + - Linking to related other `CORRADE_TARGET_*` variables, or saying it's + a superset of / implied by another + - Mentioning possible caveats (like with `LZCNT` having a dangerous `BSR` + fallback) +2. Add detection into `configure.h.cmake` + - On GCC / Clang it's usually a preprocessor variable in a form of + `__FOO__`, verify with `echo | gcc -dM -E -mfoo | grep FOO` + - If it's a superset of / implied by another, verify that the other + preprocessor variables are set / imply this option as expected + - Cross-check with Clang and if it has a different behavior drop the + implication + - On MSVC just assume it's enabled by the option that implies it, as + there it's harder to verify + - Special-case clang-cl under the MSVC branch, reusing the `__FOO__` + check for it +3. Add a new `FooT` tag type into `Cpu.h` + - Roughly at the place where it historically appeared among other + extensions (e.g. @ref Cpu::PopcntT is after `Sse42T` and before `AvxT` + but new AVX-512 extensions would go at the end) + - If a superset of / implied by another, wire it into the class + hierarchy (like e.g. @ref Cpu::AvxT), otherwise keep it standalone + (like e.g. @ref Cpu::PopcntT) + - Make the minimal documentation only link to the tag instance +4. Add a @cpp TypeTraits @ce entry. Its value is important for proper + overload resolution: + - If it's in a hierarchy, renumber the other indices so it's larger than + everything before it and smaller than everything after. Worst case, if + the hierarchy becomes larger than 16 items, the `ExtraTagBitOffset` may + need to get increased. + - If it's not, `ExtraTagCount` may need to get updated to account for the + new extra tag. You'll get static assertions if the tag value is too + small. +5. Add a new `Foo` tag: + - Position matching the order in which the tag types were defined + - Documentation similar to the `CORRADE_TARGET_FOO` docs, including the + Wikipedia link and potential caveats + - Mentioning it's either a superset of / implied by another (like e.g. + @ref Cpu::Avx) or an "extra" (like e.g. @ref Cpu::Popcnt) + - Linking to relevant other tags and corresponding `CORRADE_TARGET_FOO` + and `CORRADE_ENABLE_FOO` variables + - List it in the right place in the @ref Utility::Debug output operator + in `Cpu.cpp` --- "extra" tags go after the base ones + - If it seems like a reasonable addition, expand the `doc/cpu.dot` graph + with the new tag +6. Wire it into compile-time detection: + - Inside the @ref Cpu::DefaultBaseT / @ref Cpu::DefaultExtraT typedef and + in the @ref Cpu::compiledFeatures() function, at a place that + corresponds to where it was defined among the tags + - Mentioned in @ref Cpu::DefaultBase / @ref Cpu::DefaultExtra and + @ref Cpu::compiledFeatures() docs, again in proper order + - If it doesn't have a direct mapping to a MSVC `/arch:` option, consider + also listing it among others in the yllow warning block in @ref Cpu + namespace docs +7. Wire it into runtime detection: + - For x86 find the right [CPUID](https://en.wikipedia.org/wiki/CPUID) + bit, order the branch among checks for surrounding bits + - For ARM on Linux and Android check for a corresponding + [HWCAP](https://github.com/torvalds/linux/blame/master/arch/arm64/include/uapi/asm/hwcap.h) + bit, in some cases it may need to read the `AT_HWCAP2` entry as well + which has support further limited -- see docs in the code for more + information. + - For ARM on macOS / iOS check the corresponding + [sysctlbyname()](https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics) + value + - For WebAssembly there's no runtime detection yet. Check if anything + happened with the [feature detection proposal](https://github.com/WebAssembly/feature-detection/blob/main/proposals/feature-detection/Overview.md). + - List it in @ref Cpu::runtimeFeatures() docs, if it's depending on AVX + like e.g. @ref Cpu::Bmi1 on x86 and the dependency isn't obvious, + mention that as well +8. Add a new `CORRADE_ENABLE_FOO` macro: + - With a corresponding @cpp __attribute__((__target__("foo"))) @ce, + mention the matching `-m` option in the docs + - If the instruction set works on clang-cl without having to define + anything, then add the attribute also for clang-cl (like e.g. + @ref CORRADE_ENABLE_SSE42), if it doesn't then explicitly exclude + clang-cl such as with @ref CORRADE_ENABLE_LZCNT or + @ref CORRADE_ENABLE_AVX512F. Reflect that in the documentation --- + either saying that it expands to the target attribute also on clang-cl, + or that it isn't defined there ever. + - Verify and mention the special case with `CORRADE_TARGET_FOO` + - If it's a superset of / implied by any other `-m` option (as discovered + when implementing the `CORRADE_TARGET_FOO` detection), mention that in + the docs -- but only if it's consistent for GCC and Clang. + - Add a `_CORRADE_ENABLE_FOO` variant for the GCC multiple target + attribute workaround -- one empty for when `CORRADE_TARGET_FOO` is + defined and one just the string with a comma after (`"foo",`) +9. Add a test for the `CORRADE_ENABLE_FOO` macro: + - New `callInstructionFor()` variant for given tag type + - Annotated with the "function variant" of the macro (thus + @cpp CORRADE_ENABLE(FOO) @ce) and also with an @cpp #ifdef @ce for the + macro around. The function variant should be used in order to test both + the `_CORRADE_ENABLE_FOO` macro (on Clang before version 8 and GCC) and + `CORRADE_ENABLE_FOO` (elsewhere). + - Ideally using just that instruction alone (such as is the case with + `POPCNT`), if not then try to use an instruction set that's implied by + it so it doesn't need a second `CORRADE_ENABLE_*`. Clearly mark which + expression uses the tested instruction set. It should verify something + nontrivial (so not just a load/store) and return a non-zero value. + - Be sure to build & run on weird / broken compilers such as GCC 4.8 or + clang-cl. In case of GCC 4.8 the `Utility/Intrinsics*.h` headers may + need to get expanded to correctly pull in the intrinsics without + requiring `-mfoo` globally, see their code for more info. +10. Run `CpuTest` to verify everything: + - Compile-time detection (test with a `-march=native` build or + equivalent) + - Runtime detection, ideally on multiple platforms (AMD/Intel for x86, + phone + Apple M1 + CircleCI for ARM...) + - `CORRADE_ENABLE_*` macro (that a corresponding @cpp enableMacros() @ce + test case is run and gives a reasonable result, use CircleCI for + AVX-512 and new ARM extensions if not available locally) + - If the instruction set isn't detected as supported on a target, verify + that calling it crashes (temporarily comment out the `CORRADE_SKIP()` + in `enableMacros()`) +11. List the new `CORRADE_TARGET_FOO`, `CORRADE_ENABLE_FOO` variables and the + `Cpu::Foo` tag in `doc/changelog.dox` +12. Check the output of `CpuTest` CI jobs for sanity: + - Especially differences between Linux, Windows and macOS + - Between x86, ARM and WebAssembly + - Docker CircleCI x86 and ARM jobs tend to have very recent hardware, + OTOH macOS jobs have outdated Intel CPUs without AVX2, the output + should correspond to that + @section corrade-developers-copyright-year Checklist for updating copyright year 1. Verify there are no uncommitted changes in any repos, as that would diff --git a/doc/cpu.dot b/doc/cpu.dot new file mode 100644 index 000000000..b3f1174a2 --- /dev/null +++ b/doc/cpu.dot @@ -0,0 +1,58 @@ +/* + This file is part of Magnum. + + Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, + 2020, 2021, 2022 Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +digraph "x86 instruction family tree" { + rankdir=RL + ranksep=0.5 + + node [shape=circle style=filled margin="0.01,0.01" width=1] + + SSE2 [class="m-success"] + SSE3 [class="m-success"] + SSSE3 [class="m-success"] + SSE41 [class="m-success" label="SSE4.1"] + SSE42 [class="m-success" label="SSE4.2"] + AVX [class="m-success"] + AVX2 [class="m-success"] + AVX512F [class="m-success"] + + FMA3 [width=0.75 fontsize=13 class="m-primary"] + F16C [width=0.75 fontsize=13 class="m-primary"] + POPCNT [width=0.75 fontsize=13 margin="0,0" class="m-primary"] + LZCNT [width=0.75 fontsize=13 class="m-primary"] + BMI1 [width=0.75 fontsize=13 class="m-primary"] + + POPCNT -> SSE3 [style=invis] + LZCNT -> SSE3 [style=invis] + LZCNT -> POPCNT [style=invis] + BMI1 -> LZCNT [style=invis] + + FMA3 -> AVX [class="m-primary"] + F16C -> AVX [class="m-primary"] + + FMA3 -> F16C [style=invis] + + AVX512F -> AVX2 -> AVX -> SSE42 -> SSE41 -> SSSE3 -> SSE3 -> SSE2 [class="m-success" weight=100] +} diff --git a/doc/snippets/CMakeLists.txt b/doc/snippets/CMakeLists.txt index 875c0ac96..96a88e4f4 100644 --- a/doc/snippets/CMakeLists.txt +++ b/doc/snippets/CMakeLists.txt @@ -38,6 +38,7 @@ set_directory_properties(PROPERTIES CORRADE_USE_PEDANTIC_FLAGS ON) add_library(snippets STATIC + Corrade.cpp Containers.cpp Containers-stl.cpp Utility.cpp) @@ -63,6 +64,7 @@ if(NOT CMAKE_CXX_FLAGS MATCHES "-std=") (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9.3") OR (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS "19.10")) add_library(snippets-cpp17 STATIC + Corrade-cpp17.cpp Containers-stl17.cpp) target_link_libraries(snippets-cpp17 PRIVATE CorradeUtility) set_target_properties(snippets-cpp17 PROPERTIES @@ -94,6 +96,7 @@ endif() # TODO: causes spurious linker errors on Travis iOS build, so I'm disabling it if(CORRADE_WITH_TESTSUITE AND NOT CORRADE_TARGET_IOS) + set(THREADS_PREFER_PTHREAD_FLAG TRUE) find_package(Threads REQUIRED) # All define the same symbols so they need to be in separate libs @@ -117,14 +120,30 @@ if(CORRADE_WITH_TESTSUITE AND NOT CORRADE_TARGET_IOS) add_executable(testsuite-benchmark testsuite-benchmark.cpp) add_executable(testsuite-benchmark-custom testsuite-benchmark-custom.cpp) - target_link_libraries(testsuite-basic CorradeTestSuite) - target_link_libraries(testsuite-templated CorradeTestSuite) - target_link_libraries(testsuite-repeated CorradeTestSuite ${CMAKE_THREAD_LIBS_INIT}) - target_link_libraries(testsuite-instanced CorradeTestSuite) - target_link_libraries(testsuite-save-diagnostic CorradeTestSuite) - target_link_libraries(testsuite-iteration CorradeTestSuite) - target_link_libraries(testsuite-benchmark CorradeTestSuite) - target_link_libraries(testsuite-benchmark-custom CorradeTestSuite) + target_link_libraries(testsuite-basic PUBLIC CorradeTestSuite) + target_link_libraries(testsuite-templated PUBLIC CorradeTestSuite) + target_link_libraries(testsuite-repeated PUBLIC CorradeTestSuite Threads::Threads) + target_link_libraries(testsuite-instanced PUBLIC CorradeTestSuite) + target_link_libraries(testsuite-save-diagnostic PUBLIC CorradeTestSuite) + target_link_libraries(testsuite-iteration PUBLIC CorradeTestSuite) + target_link_libraries(testsuite-benchmark PUBLIC CorradeTestSuite) + target_link_libraries(testsuite-benchmark-custom PUBLIC CorradeTestSuite) + + if(CORRADE_TARGET_EMSCRIPTEN) + # Wasn't a problem for Emscripten <= 2.0.13 and isn't a problem for + # 3.1+, but for some reason 2.0.20 needs this. The corrade_add_test() + # macro does this automatically, but here we need to do it manually. + set_property(TARGET + testsuite-basic + testsuite-templated + testsuite-repeated + testsuite-instanced + testsuite-save-diagnostic + testsuite-iteration + testsuite-benchmark + testsuite-benchmark-custom + APPEND_STRING PROPERTY LINK_FLAGS " -s DISABLE_EXCEPTION_CATCHING=0") + endif() endif() if(CORRADE_TARGET_ANDROID) diff --git a/doc/snippets/Corrade-cpp17.cpp b/doc/snippets/Corrade-cpp17.cpp new file mode 100644 index 000000000..02f23a073 --- /dev/null +++ b/doc/snippets/Corrade-cpp17.cpp @@ -0,0 +1,56 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include "Corrade/Cpu.h" +#include "Corrade/Utility/Debug.h" + +#define DOXYGEN_ELLIPSIS(...) __VA_ARGS__ + +using namespace Corrade; + +#ifdef CORRADE_TARGET_X86 +int main() { +/* [Cpu-usage-compile-time] */ +Utility::Debug{} << "Base compiled instruction set:" << Cpu::DefaultBase; + +if constexpr(Cpu::DefaultBase >= Cpu::Avx2) { + // AVX2 code +} else { + // scalar code +} +/* [Cpu-usage-compile-time] */ + +/* [Cpu-usage-extra-compile-time] */ +Utility::Debug{} << "Base and extra instruction sets:" << Cpu::Default; + +if constexpr(Cpu::Default >= (Cpu::Avx2|Cpu::AvxFma)) { + // AVX2+FMA code +} else { + // scalar code +} +/* [Cpu-usage-extra-compile-time] */ +} +#endif diff --git a/doc/snippets/Corrade.cpp b/doc/snippets/Corrade.cpp new file mode 100644 index 000000000..dd5c11070 --- /dev/null +++ b/doc/snippets/Corrade.cpp @@ -0,0 +1,238 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include "Corrade/Cpu.h" +#include "Corrade/Containers/ArrayView.h" +#include "Corrade/Utility/Debug.h" + +#define DOXYGEN_ELLIPSIS(...) __VA_ARGS__ + +using namespace Corrade; + +#ifdef CORRADE_TARGET_X86 +/* [Cpu-usage-declare] */ +void transform(Cpu::ScalarT, Containers::ArrayView data); +void transform(Cpu::Sse42T, Containers::ArrayView data); +void transform(Cpu::Avx2T, Containers::ArrayView data); +/* [Cpu-usage-declare] */ + +/* [Cpu-usage-extra-declare] */ +int lookup(CORRADE_CPU_DECLARE(Cpu::Sse2), DOXYGEN_ELLIPSIS(int)); +int lookup(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt), DOXYGEN_ELLIPSIS(int)); +/* [Cpu-usage-extra-declare] */ +int lookup(CORRADE_CPU_DECLARE(Cpu::Scalar), DOXYGEN_ELLIPSIS(int)); +/* Might needed if Default doesn't include SSE2 on 32-bit */ + +namespace Foo { +/* [Cpu-usage-extra-ambiguity] */ +int lookup(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt), DOXYGEN_ELLIPSIS(int)); +int lookup(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Lzcnt), DOXYGEN_ELLIPSIS(int)); +/* [Cpu-usage-extra-ambiguity] */ + +int lookup(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt), int); +/* [Cpu-usage-extra-ambiguity-resolve] */ +int lookup(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt), DOXYGEN_ELLIPSIS(int)) { + // Or the other variant, or a custom third implementation ... + return lookup(CORRADE_CPU_SELECT(Cpu::Sse41|Cpu::Lzcnt), DOXYGEN_ELLIPSIS(0)); +} +/* [Cpu-usage-extra-ambiguity-resolve] */ +} + +/* [Cpu-usage-target-attributes] */ +int lookup(CORRADE_CPU_DECLARE(Cpu::Scalar), DOXYGEN_ELLIPSIS(int)) { + DOXYGEN_ELLIPSIS(return 0;) +} +#ifdef CORRADE_ENABLE_SSE2 +CORRADE_ENABLE_SSE2 int lookup(CORRADE_CPU_DECLARE(Cpu::Sse2), DOXYGEN_ELLIPSIS(int)) { + DOXYGEN_ELLIPSIS(return 0;) +} +#endif +#if defined(CORRADE_ENABLE_SSE41) && \ + defined(CORRADE_ENABLE_POPCNT) && \ + defined(CORRADE_ENABLE_LZCNT) +CORRADE_ENABLE(SSE41,POPCNT,LZCNT) int lookup( + CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt), DOXYGEN_ELLIPSIS(int)) +{ + DOXYGEN_ELLIPSIS(return 0;) +} +#endif +/* [Cpu-usage-target-attributes] */ + +namespace Bar { +using TransformT = void(*)(Containers::ArrayView); +TransformT transformImplementation(Cpu::ScalarT); +TransformT transformImplementation(Cpu::Sse42T); +TransformT transformImplementation(Cpu::Avx2T); +TransformT transformImplementation(Cpu::Features); +/* [Cpu-usage-automatic-runtime-dispatch-declare] */ +using TransformT = void(*)(Containers::ArrayView); + +TransformT transformImplementation(Cpu::ScalarT) { + return [](Containers::ArrayView data) { DOXYGEN_ELLIPSIS(static_cast(data);) }; +} +TransformT transformImplementation(Cpu::Sse42T) { + return [](Containers::ArrayView data) { DOXYGEN_ELLIPSIS(static_cast(data);) }; +} +TransformT transformImplementation(Cpu::Avx2T) { + return [](Containers::ArrayView data) { DOXYGEN_ELLIPSIS(static_cast(data);) }; +} + +CORRADE_CPU_DISPATCHER_BASE(transformImplementation) +/* [Cpu-usage-automatic-runtime-dispatch-declare] */ + +namespace Baz { +TransformT transformImplementation(Cpu::Avx2T); +/* [Cpu-usage-automatic-runtime-dispatch-target-attributes] */ +#ifdef CORRADE_ENABLE_AVX2 +CORRADE_ENABLE_AVX2 TransformT transformImplementation(Cpu::Avx2T) { + return [](Containers::ArrayView data) CORRADE_ENABLE_AVX2 { DOXYGEN_ELLIPSIS(static_cast(data);) }; +} +#endif +/* [Cpu-usage-automatic-runtime-dispatch-target-attributes] */ +} + +using LookupT = int(*)(int); +LookupT lookupImplementation(CORRADE_CPU_DECLARE(Cpu::Scalar)); +LookupT lookupImplementation(CORRADE_CPU_DECLARE(Cpu::Sse2)); +LookupT lookupImplementation(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt)); +LookupT lookupImplementation(Cpu::Features); +/* [Cpu-usage-automatic-runtime-dispatch-extra-declare] */ +using LookupT = int(*)(DOXYGEN_ELLIPSIS(int)); + +LookupT lookupImplementation(CORRADE_CPU_DECLARE(Cpu::Scalar)) { + DOXYGEN_ELLIPSIS(return {};) +} +LookupT lookupImplementation(CORRADE_CPU_DECLARE(Cpu::Sse2)) { + DOXYGEN_ELLIPSIS(return {};) +} +LookupT lookupImplementation(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt)) { + DOXYGEN_ELLIPSIS(return {};) +} + +CORRADE_CPU_DISPATCHER(lookupImplementation, Cpu::Popcnt, Cpu::Lzcnt) +/* [Cpu-usage-automatic-runtime-dispatch-extra-declare] */ + +#ifdef CORRADE_CPU_USE_IFUNC +/* [Cpu-usage-automatic-cached-dispatch-ifunc] */ +CORRADE_CPU_DISPATCHED_IFUNC(lookupImplementation, int lookup(DOXYGEN_ELLIPSIS(int))) +/* [Cpu-usage-automatic-cached-dispatch-ifunc] */ +#else +/* [Cpu-usage-automatic-cached-dispatch-pointer] */ +CORRADE_CPU_DISPATCHED_POINTER(lookupImplementation, int(*lookup)(DOXYGEN_ELLIPSIS(int))) +/* [Cpu-usage-automatic-cached-dispatch-pointer] */ +#endif + +namespace BarInsideABar { +int lookup(DOXYGEN_ELLIPSIS(int)); +/* [Cpu-usage-automatic-cached-dispatch-compile-time] */ +int lookup(DOXYGEN_ELLIPSIS(int)) { + return lookupImplementation(CORRADE_CPU_SELECT(Cpu::Default))(DOXYGEN_ELLIPSIS(0)); +} +/* [Cpu-usage-automatic-cached-dispatch-compile-time] */ +} + +} +#endif + +inline void foo(Cpu::ScalarT) {} + +int main() { +#ifdef CORRADE_TARGET_X86 +{ +Containers::ArrayView data; +/* [Cpu-usage-compile-time-call] */ +transform(Cpu::DefaultBase, data); +/* [Cpu-usage-compile-time-call] */ +} + +{ +Containers::ArrayView data; +/* [Cpu-usage-runtime-manual-dispatch] */ +Cpu::Features features = Cpu::runtimeFeatures(); +Utility::Debug{} << "Instruction set available at runtime:" << features; + +if(features & Cpu::Avx2) + transform(Cpu::Avx2, data); +else if(features & Cpu::Sse41) + transform(Cpu::Sse41, data); +else + transform(Cpu::Scalar, data); +/* [Cpu-usage-runtime-manual-dispatch] */ +} + +{ +/* [Cpu-usage-extra-compile-time-call] */ +int found = lookup(CORRADE_CPU_SELECT(Cpu::Default), DOXYGEN_ELLIPSIS(0)); +/* [Cpu-usage-extra-compile-time-call] */ +static_cast(found); +} + +{ +using namespace Bar; +Containers::ArrayView data; +/* [Cpu-usage-automatic-runtime-dispatch-call] */ +/* Dispatch once and cache the function pointer */ +TransformT transform = transformImplementation(Cpu::runtimeFeatures()); + +/* Call many times */ +transform(data); +/* [Cpu-usage-automatic-runtime-dispatch-call] */ +} + +{ +using namespace Bar; +#ifndef CORRADE_CPU_USE_IFUNC +#define LOOKUP_USES_FUNCTION_POINTER +#endif +/* [Cpu-usage-automatic-cached-dispatch-call] */ +#ifdef LOOKUP_USES_FUNCTION_POINTER +int (*lookup)(DOXYGEN_ELLIPSIS(int)); +#else +int lookup(DOXYGEN_ELLIPSIS(int)); +#endif + +int found = lookup(DOXYGEN_ELLIPSIS(0)); +/* [Cpu-usage-automatic-cached-dispatch-call] */ +static_cast(found); +} + +{ +/* [Cpu-tag-from-type] */ +foo(Cpu::Avx2); +foo(Cpu::tag()); +/* [Cpu-tag-from-type] */ +} +{ +/* [Cpu-features-from-type] */ +Cpu::Features a = Cpu::Avx2; +Cpu::Features b = Cpu::features(); +/* [Cpu-features-from-type] */ +static_cast(a); +static_cast(b); +} +#endif + +} diff --git a/modules/FindCorrade.cmake b/modules/FindCorrade.cmake index ff3429b2b..1343f3feb 100644 --- a/modules/FindCorrade.cmake +++ b/modules/FindCorrade.cmake @@ -80,6 +80,9 @@ # CORRADE_BUILD_MULTITHREADED - Defined if compiled in a way that makes it # possible to safely use certain Corrade features simultaneously in multiple # threads +# CORRADE_BUILD_CPU_RUNTIME_DISPATCH - Defined if built with code paths +# optimized for multiple architectres with the best matching variant selected +# at runtime based on detected CPU features # CORRADE_TARGET_UNIX - Defined if compiled for some Unix flavor # (Linux, BSD, macOS) # CORRADE_TARGET_APPLE - Defined if compiled for Apple platforms @@ -100,6 +103,8 @@ # CORRADE_TARGET_MSVC - Defined if compiling with MSVC or Clang with # a MSVC frontend # CORRADE_TARGET_MINGW - Defined if compiling under MinGW +# CORRADE_CPU_USE_IFUNC - Defined if GNU IFUNC is allowed to be used +# for runtime dispatch in the Cpu library # CORRADE_PLUGINMANAGER_NO_DYNAMIC_PLUGIN_SUPPORT - Defined if PluginManager # doesn't support dynamic plugin loading due to platform limitations # CORRADE_TESTSUITE_TARGET_XCTEST - Defined if TestSuite is targeting Xcode @@ -319,6 +324,7 @@ set(_corradeFlags BUILD_STATIC BUILD_STATIC_UNIQUE_GLOBALS BUILD_MULTITHREADED + BUILD_CPU_RUNTIME_DISPATCH TARGET_UNIX TARGET_APPLE TARGET_IOS @@ -327,10 +333,12 @@ set(_corradeFlags TARGET_WINDOWS_RT TARGET_EMSCRIPTEN TARGET_ANDROID - # TARGET_X86 etc and TARGET_LIBCXX are not exposed to CMake as the meaning - # is unclear on platforms with multi-arch binaries or when mixing different - # STL implementations. TARGET_GCC etc are figured out via UseCorrade.cmake, - # as the compiler can be different when compiling the lib & when using it. + # TARGET_X86 etc, TARGET_32BIT, TARGET_BIG_ENDIAN and TARGET_LIBCXX etc. + # are not exposed to CMake as the meaning is unclear on platforms with + # multi-arch binaries or when mixing different STL implementations. + # TARGET_GCC etc are figured out via UseCorrade.cmake, as the compiler can + # be different when compiling the lib & when using it. + CPU_USE_IFUNC PLUGINMANAGER_NO_DYNAMIC_PLUGIN_SUPPORT TESTSUITE_TARGET_XCTEST UTILITY_USE_ANSI_COLORS) diff --git a/modules/FindNodeJs.cmake b/modules/FindNodeJs.cmake index 8823e6167..3169eb94b 100644 --- a/modules/FindNodeJs.cmake +++ b/modules/FindNodeJs.cmake @@ -6,6 +6,7 @@ # # NodeJs_FOUND - True if Node.js executable is found # NodeJs::NodeJs - Node.js executable imported target +# NodeJs_VERSION - Version string reported by ``node --version`` # # @@ -37,8 +38,21 @@ find_program(NODEJS_EXECUTABLE node) mark_as_advanced(NODEJS_EXECUTABLE) +if(NODEJS_EXECUTABLE) + execute_process(COMMAND ${NODEJS_EXECUTABLE} --version + OUTPUT_VARIABLE NodeJs_VERSION + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + if(NodeJs_VERSION MATCHES "^v[0-9]") + string(SUBSTRING ${NodeJs_VERSION} 1 -1 NodeJs_VERSION) + else() + unset(NodeJs_VERSION) + endif() +endif() + include(FindPackageHandleStandardArgs) -find_package_handle_standard_args("NodeJs" DEFAULT_MSG NODEJS_EXECUTABLE) +find_package_handle_standard_args(NodeJs + REQUIRED_VARS NODEJS_EXECUTABLE + VERSION_VAR NodeJs_VERSION) if(NOT TARGET NodeJs::NodeJs) add_executable(NodeJs::NodeJs IMPORTED) diff --git a/modules/UseCorrade.cmake b/modules/UseCorrade.cmake index af1726e68..016104ea5 100644 --- a/modules/UseCorrade.cmake +++ b/modules/UseCorrade.cmake @@ -346,6 +346,10 @@ function(corrade_add_test test_name) set(_corrade_file_pair_match "^(.+)@([^@]+)$") set(_corrade_file_pair_replace "\\1;\\2") + # TestSuite library to link to. Gets reset below if the tests already link + # to their own variant. + set(testsuite_library Corrade::TestSuite) + # Get DLL and path lists foreach(arg ${ARGN}) if(arg STREQUAL LIBRARIES) @@ -359,6 +363,12 @@ function(corrade_add_test test_name) set(_DOING_ARGUMENTS ON) else() if(_DOING_LIBRARIES) + # If Corrade's own tests link their own variant of TestSuite, + # don't link the implicit one as well, as otherwise we'd end + # up with duplicated symbols. + if(arg STREQUAL CorradeTestSuiteTestLib) + set(testsuite_library ) + endif() list(APPEND libraries ${arg}) elseif(_DOING_FILES) # If the file is already a pair of file and destination, just @@ -404,7 +414,7 @@ function(corrade_add_test test_name) add_library(${test_name} SHARED ${sources}) set_target_properties(${test_name} PROPERTIES FRAMEWORK TRUE) # This is never Windows, so no need to bother with Corrade::Main - target_link_libraries(${test_name} PRIVATE ${libraries} Corrade::TestSuite) + target_link_libraries(${test_name} PRIVATE ${libraries} ${testsuite_library}) set(test_runner_file ${CMAKE_CURRENT_BINARY_DIR}/${test_name}.mm) configure_file(${CORRADE_TESTSUITE_XCTEST_RUNNER} @@ -431,7 +441,7 @@ function(corrade_add_test test_name) endif() else() add_executable(${test_name} ${sources}) - target_link_libraries(${test_name} PRIVATE ${libraries} Corrade::TestSuite Corrade::Main) + target_link_libraries(${test_name} PRIVATE ${libraries} ${testsuite_library} Corrade::Main) # Run tests using Node.js on Emscripten if(CORRADE_TARGET_EMSCRIPTEN) @@ -440,7 +450,22 @@ function(corrade_add_test test_name) set_property(TARGET ${test_name} APPEND_STRING PROPERTY COMPILE_FLAGS " -s DISABLE_EXCEPTION_CATCHING=0") set_property(TARGET ${test_name} APPEND_STRING PROPERTY LINK_FLAGS " -s DISABLE_EXCEPTION_CATCHING=0") find_package(NodeJs REQUIRED) - add_test(NAME ${test_name} COMMAND NodeJs::NodeJs $ ${arguments}) + # Node.js before version 17 needs --experimental-wasm-simd. Version + # 17 upgraded to V8 9.1, which has the option enabled by default: + # https://github.com/nodejs/node/commit/a7cbf19a82c75e9a65e90fb8ba4947e2fc52ef39 + # Since the code defining these flags explicitly says "remove once + # they hit stable": + # https://github.com/v8/v8/blob/ba8ad5dd17ea85c856c09c2ff603641487d1f0ca/src/wasm/wasm-feature-flags.h#L101-L109 + # even though it causes numberous issues such as: + # https://github.com/nodejs/node/issues/43592 + # I'm future-proofing and not setting it for version 17+. From the + # other side, the flag goes back to ancient version 6 (2017), so it + # should be no problem to just pass it there always: + # https://github.com/v8/v8/commit/45618a9ab5cc98a5de200b0116670e8c272f0c5f + if(NodeJS_VERSION VERSION_LESS 17) + set(experimental_wasm_simd --experimental-wasm-simd) + endif() + add_test(NAME ${test_name} COMMAND NodeJs::NodeJs ${experimental_wasm_simd} $ ${arguments}) # Embed all files foreach(file ${files}) diff --git a/package/ci/circleci.yml b/package/ci/circleci.yml index 41aec4405..a2c07963b 100644 --- a/package/ci/circleci.yml +++ b/package/ci/circleci.yml @@ -19,6 +19,14 @@ executors: # filesystem race and randomly complains that a file doesn't exist: # https://github.com/mosra/magnum/issues/413, # https://github.com/emscripten-core/emscripten/pull/10161 + # + # Regarding SIMD, while 1.39.6 supports some WIP variant of it, the + # finalized support is only since 2.0.18 (and then the first non-broken + # emsdk is 2.0.25). But emsdk ships with Node.js 14, and only Node.js 15 + # supports intrinsics that Emscripten 2.0.18 can produce, so there's no + # point in even trying, we wouldn't be able to run the tests anyway. + # TODO: revisit when https://github.com/emscripten-core/emsdk/issues/1064 + # or any other referenced issues are finally resolved - image: emscripten/emsdk:1.39.6-upstream python-3_6: docker: diff --git a/src/Corrade/CMakeLists.txt b/src/Corrade/CMakeLists.txt index 9223e6bb3..18766c818 100644 --- a/src/Corrade/CMakeLists.txt +++ b/src/Corrade/CMakeLists.txt @@ -80,6 +80,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/version.h.cmake set(Corrade_HEADERS Corrade.h + Cpu.h Tags.h) # Force IDEs to display all header files in project view diff --git a/src/Corrade/Containers/StringView.cpp b/src/Corrade/Containers/StringView.cpp index 3160897e0..64db6af34 100644 --- a/src/Corrade/Containers/StringView.cpp +++ b/src/Corrade/Containers/StringView.cpp @@ -30,6 +30,7 @@ #include #include +#include "Corrade/Cpu.h" #include "Corrade/Containers/Array.h" #include "Corrade/Containers/ArrayView.h" #include "Corrade/Containers/GrowableArray.h" @@ -39,6 +40,16 @@ #include "Corrade/Utility/DebugStl.h" #include "Corrade/Utility/Math.h" +#if (defined(CORRADE_ENABLE_SSE2) || defined(CORRADE_ENABLE_AVX)) && defined(CORRADE_ENABLE_BMI1) +#include "Corrade/Utility/IntrinsicsAvx.h" /* TZCNT is in AVX headers :( */ +#endif +#ifdef CORRADE_ENABLE_NEON +#include +#endif +#ifdef CORRADE_ENABLE_SIMD128 +#include +#endif + namespace Corrade { namespace Containers { template BasicStringView::BasicStringView(T* const data, const StringViewFlags flags, std::nullptr_t) noexcept: BasicStringView{data, @@ -97,9 +108,9 @@ template Array> BasicStringView::splitWithoutEmpt return parts; } -namespace { +namespace Implementation { -inline const char* find(const char* data, const std::size_t size, const char* const substring, const std::size_t substringSize) { +const char* stringFindString(const char* data, const std::size_t size, const char* const substring, const std::size_t substringSize) { /* If the substring is not larger than the string we search in */ if(substringSize <= size) { /* If these are both empty (substringSize <= size, so it's also 0), @@ -121,7 +132,7 @@ inline const char* find(const char* data, const std::size_t size, const char* co return {}; } -inline const char* findLast(const char* const data, const std::size_t size, const char* const substring, const std::size_t substringSize) { +const char* stringFindLastString(const char* const data, const std::size_t size, const char* const substring, const std::size_t substringSize) { /* If the substring is not larger than the string we search in */ if(substringSize <= size) { /* If these are both empty (substringSize <= size, so it's also 0), @@ -143,14 +154,555 @@ inline const char* findLast(const char* const data, const std::size_t size, cons return {}; } -inline const char* find(const char* data, const std::size_t size, const char character) { - /* Making a utility function because yet again I'm not sure if null - pointers are allowed and cppreference says nothing about that, so in - case this needs to be patched it's better to have it in a single place */ +namespace { + +/* SIMD implementation of character lookup. Loosely based off + https://docs.rs/memchr/2.3.4/src/memchr/x86/sse2.rs.html, which in turn is + based off https://gms.tf/stdfind-and-memchr-optimizations.html, which at the + time of writing (Jul 2022) uses m.css, so the circle is complete :)) + + The code below is commented, but the core points are the following: + + 1. do as much as possible via aligned loads, + 2. otherwise, do as much as possible via unaligned vector loads even at + the cost of ovelapping with an aligned load, + 3. otherwise, fall back to a smaller vector width (AVX -> SSE) or to a + scalar code + + The 128-bit variant first checks if there's less than 16 bytes. If it is, it + just checks each of them sequentially. Otherwise, with 16 and more bytes, + the following is done: + + +---+ +---+ + | A | | D | + +---+ +---+ + +---+---+---+---+ +---+-- + | B : : : | ... | C | ... + +---+---+---+---+ +---+-- + + A. First it does an unconditional unaligned load of a single vector + (assuming an extra conditional branch would likely be slower than the + unaligned load ovehead), compares all bytes inside to the (broadcasted) + search value and for all bytes that are equal calculates a bitmask (if + 4th and 7th byte is present, the bitmask has bit 4 and 7 set). Then, if + any bit is set, returns the position of the first bit which is the + found index. + B. Next it finds an aligned position. If the vector A was already aligned, + it will start right after, otherwise there may be up to 15 bytes + overlap that'll be checked twice. From the aligned position, to avoid + branching too often, it goes in a batch of four vectors at a time, + checking the result together for all four. Which also helps offset the + extra work from the initial overlap. + C. Once there is less than four vectors left, it goes vector-by-vector, + still doing aligned loads, but branching for every. + D. Once there's less than 16 bytes left, it performs an unaligned load + that may overlap with the previous aligned vector, similarly to the + initial unaligned load A. + + The 256-bit variant is mostly just about expanding from 16 bytes at a time + to 32 bytes at a time. The only difference is that instead of doing a + scalar fallback for less than 32 bytes, it delegates to the 128-bit + variant --- effectively performing the lookup with either two overlapping + 16-byte vectors (or falling back to scalar for less than 16 bytes). + + The ARM variant has the high-level concept similar to x86, except that NEON + doesn't have a bitmask instruction. Instead a "right shift and narrow" + instruction is used, see comments there for details. + + The WASM variant is mostly a direct translation of the x86 variant, except + as noted in code comments. */ + +#if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_BMI1) +CORRADE_ENABLE(SSE2,BMI1) CORRADE_ALWAYS_INLINE const char* findCharacterSingleVectorUnaligned(Cpu::Sse2T, const char* at, const __m128i vn1) { + /* _mm_lddqu_si128 is just an alias to _mm_loadu_si128 on all CPUs with + SSSE3+, no reason to use it: https://stackoverflow.com/a/38383624 */ + const __m128i chunk = _mm_loadu_si128(reinterpret_cast(at)); + if(const int mask = _mm_movemask_epi8(_mm_cmpeq_epi8(chunk, vn1))) + return at + _tzcnt_u32(mask); + return {}; +} +CORRADE_ENABLE(SSE2,BMI1) CORRADE_ALWAYS_INLINE const char* findCharacterSingleVector(Cpu::Sse2T, const char* at, const __m128i vn1) { + CORRADE_INTERNAL_DEBUG_ASSERT(reinterpret_cast(at) % 16 == 0); + + const __m128i chunk = _mm_load_si128(reinterpret_cast(at)); + if(const int mask = _mm_movemask_epi8(_mm_cmpeq_epi8(chunk, vn1))) + return at + _tzcnt_u32(mask); + return {}; +} + +CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(SSE2,BMI1) typename std::decay::type stringFindCharacterImplementation(CORRADE_CPU_DECLARE(Cpu::Sse2|Cpu::Bmi1)) { + /* Can't use trailing return type due to a GCC 9.3 bug, which is the default + on Ubuntu 20.04: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90333 */ + return [](const char* const data, const std::size_t size, const char character) CORRADE_ENABLE(SSE2,BMI1) { + const char* const end = data + size; + + /* If we have less than 16 bytes, do it the stupid way. Compared to a plain + loop this is 1.5-2x faster when unrolled. Interestingly enough, on GCC + (11) doing a pre-increment and `return j` leads to + lea 0x1(%rcx),%rax + mov %rax,%r8 + cmp 0x1(%rcx),%dl + je 0x63f43 <+243> + repeated 15 times (with <+243> returning %r8 for all), while a + post-increment and `return j - 1` is just + lea 0x1(%rax),%rcx + cmp (%rax),%dl + je 0x63f20 <+208> + with %rax and %rcx alternating in every case and the jump always + different. That's 25% instructions less for the post-increment, and the + benchmark confirms that (~3.50 vs ~2.80 µs). Clang (13) does a similar + thing, although it has `lea, cmp, mov, je` in the first case instead and + `cmp, je, add` in the second case instead, and (probably due to the + different order?) the benchmark doesn't show any difference between the + two. Since post-increment significantly helps GCC and doesn't make + Clang slower, use it. */ + { + const char* j = data; + switch(size) { + case 15: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 14: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 13: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 12: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 11: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 10: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 9: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 8: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 7: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 6: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 5: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 4: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 3: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 2: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 1: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 0: return static_cast(nullptr); + } + } + + const __m128i vn1 = _mm_set1_epi8(character); + + /* Unconditionally do a lookup in the first vector a slower, unaligned + way. Any extra branching to avoid the unaligned load if already aligned + would be most probably more expensive than the actual unaligned load. */ + if(const char* const found = findCharacterSingleVectorUnaligned(Cpu::Sse2, data, vn1)) + return found; + + /* Go to the next aligned position. If the pointer was already aligned, + we'll go to the next aligned vector; if not, there will be an overlap + and we'll check some bytes twice. */ + const char* i = reinterpret_cast(reinterpret_cast(data + 16) & ~0xf); + CORRADE_INTERNAL_DEBUG_ASSERT(i >= data && reinterpret_cast(i) % 16 == 0); + + /* Go four vectors at a time with the aligned pointer */ + for(; i + 4*16 < end; i += 4*16) { + const __m128i a = _mm_load_si128(reinterpret_cast(i) + 0); + const __m128i b = _mm_load_si128(reinterpret_cast(i) + 1); + const __m128i c = _mm_load_si128(reinterpret_cast(i) + 2); + const __m128i d = _mm_load_si128(reinterpret_cast(i) + 3); + + const __m128i eqa = _mm_cmpeq_epi8(vn1, a); + const __m128i eqb = _mm_cmpeq_epi8(vn1, b); + const __m128i eqc = _mm_cmpeq_epi8(vn1, c); + const __m128i eqd = _mm_cmpeq_epi8(vn1, d); + + const __m128i or1 = _mm_or_si128(eqa, eqb); + const __m128i or2 = _mm_or_si128(eqc, eqd); + const __m128i or3 = _mm_or_si128(or1, or2); + if(_mm_movemask_epi8(or3)) { + if(const int mask = _mm_movemask_epi8(eqa)) + return i + 0*16 + _tzcnt_u32(mask); + if(const int mask = _mm_movemask_epi8(eqb)) + return i + 1*16 + _tzcnt_u32(mask); + if(const int mask = _mm_movemask_epi8(eqc)) + return i + 2*16 + _tzcnt_u32(mask); + if(const int mask = _mm_movemask_epi8(eqd)) + return i + 3*16 + _tzcnt_u32(mask); + CORRADE_INTERNAL_DEBUG_ASSERT_UNREACHABLE(); /* LCOV_EXCL_LINE */ + } + } + + /* Handle remaining less than four vectors */ + for(; i + 16 <= end; i += 16) + if(const char* const found = findCharacterSingleVector(Cpu::Sse2, i, vn1)) + return found; + + /* Handle remaining less than a vector with an unaligned search, again + overlapping back with the previous already-searched elements */ + if(i < end) { + CORRADE_INTERNAL_DEBUG_ASSERT(i + 16 > end); + i = end - 16; + return findCharacterSingleVectorUnaligned(Cpu::Sse2, i, vn1); + } + + return static_cast(nullptr); + }; +} +#endif + +#if defined(CORRADE_ENABLE_AVX2) && defined(CORRADE_ENABLE_BMI1) +CORRADE_ENABLE(AVX2,BMI1) CORRADE_ALWAYS_INLINE const char* findCharacterSingleVectorUnaligned(Cpu::Avx2T, const char* at, const __m256i vn1) { + /* _mm256_lddqu_si256 is just an alias to _mm256_loadu_si256, no reason to + use it: https://stackoverflow.com/a/47426790 */ + const __m256i chunk = _mm256_loadu_si256(reinterpret_cast(at)); + if(const int mask = _mm256_movemask_epi8(_mm256_cmpeq_epi8(chunk, vn1))) + return at + _tzcnt_u32(mask); + return {}; +} +CORRADE_ENABLE(AVX2,BMI1) CORRADE_ALWAYS_INLINE const char* findCharacterSingleVector(Cpu::Avx2T, const char* at, const __m256i vn1) { + CORRADE_INTERNAL_DEBUG_ASSERT(reinterpret_cast(at) % 32 == 0); + + const __m256i chunk = _mm256_load_si256(reinterpret_cast(at)); + if(const int mask = _mm256_movemask_epi8(_mm256_cmpeq_epi8(chunk, vn1))) + return at + _tzcnt_u32(mask); + return {}; +} + +CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(AVX2,BMI1) typename std::decay::type stringFindCharacterImplementation(CORRADE_CPU_DECLARE(Cpu::Avx2|Cpu::Bmi1)) { + /* Can't use trailing return type due to a GCC 9.3 bug, which is the default + on Ubuntu 20.04: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90333 */ + return [](const char* const data, const std::size_t size, const char character) CORRADE_ENABLE(AVX2,BMI1) { + const char* const end = data + size; + + /* If we have less than 32 bytes, fall back to the SSE variant */ + /** @todo deinline it here? any speed gains from rewriting using 128-bit + AVX? or does the compiler do that automatically? */ + if(size < 32) + return stringFindCharacterImplementation(CORRADE_CPU_SELECT(Cpu::Sse2|Cpu::Bmi1))(data, size, character); + + const __m256i vn1 = _mm256_set1_epi8(character); + + /* Unconditionally do a lookup in the first vector a slower, unaligned + way. Any extra branching to avoid the unaligned load if already aligned + would be most probably more expensive than the actual unaligned load. */ + /** @todo not great, slower than calling SSE directly :( */ + if(const char* const found = findCharacterSingleVectorUnaligned(Cpu::Avx2, data, vn1)) + return found; + + /* Go to the next aligned position. If the pointer was already aligned, + we'll go to the next aligned vector; if not, there will be an overlap + and we'll check some bytes twice. */ + const char* i = reinterpret_cast(reinterpret_cast(data + 32) & ~0x1f); + CORRADE_INTERNAL_DEBUG_ASSERT(i >= data && reinterpret_cast(i) % 32 == 0); + + /* Go four vectors at a time with the aligned pointer */ + for(; i + 4*32 < end; i += 4*32) { + const __m256i a = _mm256_load_si256(reinterpret_cast(i) + 0); + const __m256i b = _mm256_load_si256(reinterpret_cast(i) + 1); + const __m256i c = _mm256_load_si256(reinterpret_cast(i) + 2); + const __m256i d = _mm256_load_si256(reinterpret_cast(i) + 3); + + const __m256i eqa = _mm256_cmpeq_epi8(vn1, a); + const __m256i eqb = _mm256_cmpeq_epi8(vn1, b); + const __m256i eqc = _mm256_cmpeq_epi8(vn1, c); + const __m256i eqd = _mm256_cmpeq_epi8(vn1, d); + + const __m256i or1 = _mm256_or_si256(eqa, eqb); + const __m256i or2 = _mm256_or_si256(eqc, eqd); + const __m256i or3 = _mm256_or_si256(or1, or2); + if(_mm256_movemask_epi8(or3)) { + /** @todo exploit the TZCNT property of returning 32 for zero + input somehow? trivial sum would work only if there's at most + one found byte among all 128 */ + if(const int mask = _mm256_movemask_epi8(eqa)) + return i + 0*32 + _tzcnt_u32(mask); + if(const int mask = _mm256_movemask_epi8(eqb)) + return i + 1*32 + _tzcnt_u32(mask); + if(const int mask = _mm256_movemask_epi8(eqc)) + return i + 2*32 + _tzcnt_u32(mask); + if(const int mask = _mm256_movemask_epi8(eqd)) + return i + 3*32 + _tzcnt_u32(mask); + CORRADE_INTERNAL_DEBUG_ASSERT_UNREACHABLE(); /* LCOV_EXCL_LINE */ + } + } + + /* Handle remaining less than four vectors */ + for(; i + 32 <= end; i += 32) + if(const char* const found = findCharacterSingleVector(Cpu::Avx2, i, vn1)) + return found; + + /* Handle remaining less than a vector with an unaligned search, again + overlapping back with the previous already-searched elements */ + if(i < end) { + CORRADE_INTERNAL_DEBUG_ASSERT(i + 32 > end); + i = end - 32; + return findCharacterSingleVectorUnaligned(Cpu::Avx2, i, vn1); + } + + return static_cast(nullptr); + }; +} +#endif + +#ifdef CORRADE_ENABLE_NEON +/* AArch64 doesn't differentiate between aligned and unaligned loads. ARM32 + does, but it's not exposed in the intrinsics, only in compiler-specific + ways. Since 32-bit ARM is increasingly rare, not bothering at all. + https://stackoverflow.com/a/53245244 */ +CORRADE_ENABLE(NEON) CORRADE_ALWAYS_INLINE const char* findCharacterSingleVectorUnaligned(Cpu::NeonT, const char* at, const uint8x16_t vn1) { + const uint8x16_t chunk = vld1q_u8(reinterpret_cast(at)); + + /* Emulating _mm_movemask_epi8() on ARM is rather expensive, even the most + optimized variant listed at https://github.com/WebAssembly/simd/pull/201 + is 6+ instructions. Instead, a "shift right and narrow" is used, based + on an idea from https://twitter.com/Danlark1/status/1539344281336422400 + and further explained in https://github.com/facebook/zstd/pull/3139. + + First, similarly to x86, an equivalence mask is calculated with bytes + being either ff or 00 based on whether they match: + + 00 ff ff 00 00 00 ff ff 00 00 00 00 ff 00 00 00 + + The result is reinterpreted as 8 16bit values: + + 00ff ff00 0000 ffff 0000 0000 ff00 0000 + + Then, the vshrn_n_u16() instruction shifts each 16bit value four bits to + the right, and drops the high half: + + 000f 0ff0 0000 0fff 0000 0000 0ff0 0000 + 0f f0 00 ff 00 00 f0 00 + + The result, stored in the lower half of a 128-bit register, is then + extracted as a single 64-bit number: + + 0ff0 00ff 0000 f000 + + This effectively reduces the original 128-bit mask to a half, with every + four bits describing a masked byte. While that's still 4x more than what + _mm_movemask_epi8() produces, it can be tested against zero using + regular scalar operations. Finally, `__builtin_ctzll(mask) >> 2` is + equivalent to what TZCNT on a 16bit mask produced by _mm_movemask_epi8() + would return -- there's simply just 4x more bits. */ + const uint16x8_t eq16 = vreinterpretq_u16_u8(vceqq_u8(chunk, vn1)); + const uint64x1_t shrn64 = vreinterpret_u64_u8(vshrn_n_u16(eq16, 4)); + if(const uint64_t mask = vget_lane_u64(shrn64, 0)) + return at + (__builtin_ctzll(mask) >> 2); + return {}; +} + +CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(NEON) typename std::decay::type stringFindCharacterImplementation(CORRADE_CPU_DECLARE(Cpu::Neon)) { + /* Can't use trailing return type due to a GCC 9.3 bug, which is the default + on Ubuntu 20.04: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90333 */ + return [](const char* const data, const std::size_t size, const char character) CORRADE_ENABLE(NEON) { + const char* const end = data + size; + + /* If we have less than 16 bytes, do it the stupid way. Unlike x86 or WASM, + unrolling the loop here makes things actually worse. */ + /** @todo investigate why */ + if(size < 16) { + for(const char* i = data; i != end; ++i) + if(*i == character) return i; + return static_cast(nullptr); + } + + const uint8x16_t vn1 = vdupq_n_u8(character); + + /* Unconditionally do a lookup in the first vector a slower, unaligned + way. Any extra branching to avoid the unaligned load if already aligned + would be most probably more expensive than the actual unaligned load. */ + if(const char* const found = findCharacterSingleVectorUnaligned(Cpu::Neon, data, vn1)) + return found; + + /* Go to the next aligned position. If the pointer was already aligned, + we'll go to the next aligned vector; if not, there will be an overlap + and we'll check some bytes twice. */ + const char* i = reinterpret_cast(reinterpret_cast(data + 16) & ~0xf); + CORRADE_INTERNAL_DEBUG_ASSERT(i >= data && reinterpret_cast(i) % 16 == 0); + + /* Go four vectors at a time with the aligned pointer */ + for(; i + 4*16 < end; i += 4*16) { + /** @todo https://branchfree.org/2019/04/01/fitting-my-head-through-the-arm-holes-or-two-sequences-to-substitute-for-the-missing-pmovmskb-instruction-on-arm-neon/#comment-1768 + suggests an interleaved vld4q8_u8() load instead of four separate + loads, and a sequence of vsriq_n_u8() that forms a single 64-bit + mask. Unfortunately that's actually slower than what i have here + (on Huawei P10 at least), maybe it'd be faster on newer archs? */ + + const uint8x16_t a = vld1q_u8(reinterpret_cast(i) + 0*16); + const uint8x16_t b = vld1q_u8(reinterpret_cast(i) + 1*16); + const uint8x16_t c = vld1q_u8(reinterpret_cast(i) + 2*16); + const uint8x16_t d = vld1q_u8(reinterpret_cast(i) + 3*16); + + const uint8x16_t eqa = vceqq_u8(vn1, a); + const uint8x16_t eqb = vceqq_u8(vn1, b); + const uint8x16_t eqc = vceqq_u8(vn1, c); + const uint8x16_t eqd = vceqq_u8(vn1, d); + + /* Similar to findCharacterSingleVectorUnaligned(Cpu::NeonT), except + that four "shift right and narrow" operations are done, interleaving + the result into two registers instead of four */ + const uint8x8_t maska = vshrn_n_u16(vreinterpretq_u16_u8(eqa), 4); + const uint8x16_t maskab = vshrn_high_n_u16(maska, vreinterpretq_u16_u8(eqb), 4); + const uint8x8_t maskc = vshrn_n_u16(vreinterpretq_u16_u8(eqc), 4); + const uint8x16_t maskcd = vshrn_high_n_u16(maskc, vreinterpretq_u16_u8(eqd), 4); + + /* Which makes it possible to test with just one OR and a horizontal + add instead of three ORs and a horizontal add */ + if(vaddvq_u8(vorrq_u8(maskab, maskcd))) { + if(const std::uint64_t mask = vgetq_lane_u64(vreinterpretq_u64_u8(maskab), 0)) + return i + 0*16 + (__builtin_ctzll(mask) >> 2); + if(const std::uint64_t mask = vgetq_lane_u64(vreinterpretq_u64_u8(maskab), 1)) + return i + 1*16 + (__builtin_ctzll(mask) >> 2); + if(const std::uint64_t mask = vgetq_lane_u64(vreinterpretq_u64_u8(maskcd), 0)) + return i + 2*16 + (__builtin_ctzll(mask) >> 2); + if(const std::uint64_t mask = vgetq_lane_u64(vreinterpretq_u64_u8(maskcd), 1)) + return i + 3*16 + (__builtin_ctzll(mask) >> 2); + CORRADE_INTERNAL_DEBUG_ASSERT_UNREACHABLE(); /* LCOV_EXCL_LINE */ + } + } + + /* Handle remaining less than four vectors */ + for(; i + 16 <= end; i += 16) + if(const char* const found = findCharacterSingleVectorUnaligned(Cpu::Neon, i, vn1)) + return found; + + /* Handle remaining less than a vector with an unaligned search, again + overlapping back with the previous already-searched elements */ + if(i < end) { + CORRADE_INTERNAL_DEBUG_ASSERT(i + 16 > end); + i = end - 16; + return findCharacterSingleVectorUnaligned(Cpu::Neon, i, vn1); + } + + return static_cast(nullptr); + }; +} +#endif + +#ifdef CORRADE_ENABLE_SIMD128 +/* WASM doesn't differentiate between aligned and unaligned load, it's always + unaligned :( */ +CORRADE_ENABLE_SIMD128 CORRADE_ALWAYS_INLINE const char* findCharacterSingleVectorUnaligned(Cpu::Simd128T, const char* at, const v128_t vn1) { + const v128_t chunk = wasm_v128_load(at); + if(const int mask = wasm_i8x16_bitmask(wasm_i8x16_eq(chunk, vn1))) + return at + __builtin_ctz(mask); + return {}; +} + +CORRADE_UTILITY_CPU_MAYBE_UNUSED typename std::decay::type stringFindCharacterImplementation(CORRADE_CPU_DECLARE(Cpu::Simd128)) { + return [](const char* const data, const std::size_t size, const char character) CORRADE_ENABLE_SIMD128 -> const char* { + const char* const end = data + size; + + /* If we have less than 16 bytes, do it the stupid way. Compared to a plain + loop, this is 25% faster when unrolled. Strangely enough, if the switch + is put into an external always inline function to avoid duplication with + the SSE2 variant, it no longer gives the advantage. Furthermore, the + post-increment optimization from the x86 case doesn't help here at all, + on the contrary makes the code slightly slower. */ + { + const char* j = data - 1; + switch(size) { + case 15: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 14: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 13: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 12: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 11: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 10: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 9: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 8: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 7: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 6: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 5: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 4: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 3: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 2: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 1: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 0: return {}; + } + } + + const v128_t vn1 = wasm_i8x16_splat(character); + + /* Unconditionally do a lookup in the first vector a slower, unaligned + way. Any extra branching to avoid the unaligned load if already aligned + would be most probably more expensive than the actual unaligned load. */ + if(const char* const found = findCharacterSingleVectorUnaligned(Cpu::Simd128, data, vn1)) + return found; + + /* Go to the next aligned position. If the pointer was already aligned, + we'll go to the next aligned vector; if not, there will be an overlap + and we'll check some bytes twice. */ + const char* i = reinterpret_cast(reinterpret_cast(data + 16) & ~0xf); + CORRADE_INTERNAL_DEBUG_ASSERT(i >= data && reinterpret_cast(i) % 16 == 0); + + /* Go four vectors at a time with the aligned pointer */ + for(; i + 4*16 < end; i += 4*16) { + const v128_t a = wasm_v128_load(reinterpret_cast(i) + 0); + const v128_t b = wasm_v128_load(reinterpret_cast(i) + 1); + const v128_t c = wasm_v128_load(reinterpret_cast(i) + 2); + const v128_t d = wasm_v128_load(reinterpret_cast(i) + 3); + + const v128_t eqa = wasm_i8x16_eq(vn1, a); + const v128_t eqb = wasm_i8x16_eq(vn1, b); + const v128_t eqc = wasm_i8x16_eq(vn1, c); + const v128_t eqd = wasm_i8x16_eq(vn1, d); + + const v128_t or1 = wasm_v128_or(eqa, eqb); + const v128_t or2 = wasm_v128_or(eqc, eqd); + const v128_t or3 = wasm_v128_or(or1, or2); + /* wasm_i8x16_bitmask(or3) maps directly to the SSE2 variant and is + thus fast on x86, but on ARM wasm_v128_any_true(or3) is faster. With + StringViewBenchmark::findCharacterRare() and runtime dispatch + disabled for tests, on x86 (node.js 17.8) bitmask is ~1.35 µs and + any_true ~1.85 µs; on ARM (Huawei P10, Vivaldi w/ Chromium 102) + bitmask is 14.3 µs and any_true 11.7 µs. Ideally we'd have two + runtime versions, one picking x86-friendly instructions and the + other ARM-friendly, but function pointer dispatch has a *massive* + overhead currently. Related info about instruction complexity: + https://github.com/WebAssembly/simd/pull/201 + https://github.com/zeux/wasm-simd/blob/master/Instructions.md */ + /** @todo revisit once runtime dispatch overhead gets better or once + compile-time tuning such as CORRADE_TARGET_WASM_SIMD128_ARM / _X86 + exists */ + if(wasm_i8x16_bitmask(or3)) { + if(const int mask = wasm_i8x16_bitmask(eqa)) + return i + 0*16 + __builtin_ctz(mask); + if(const int mask = wasm_i8x16_bitmask(eqb)) + return i + 1*16 + __builtin_ctz(mask); + if(const int mask = wasm_i8x16_bitmask(eqc)) + return i + 2*16 + __builtin_ctz(mask); + if(const int mask = wasm_i8x16_bitmask(eqd)) + return i + 3*16 + __builtin_ctz(mask); + CORRADE_INTERNAL_DEBUG_ASSERT_UNREACHABLE(); /* LCOV_EXCL_LINE */ + } + } + + /* Handle remaining less than four vectors */ + for(; i + 16 <= end; i += 16) + if(const char* const found = findCharacterSingleVectorUnaligned(Cpu::Simd128, i, vn1)) + return found; + + /* Handle remaining less than a vector with an unaligned search, again + overlapping back with the previous already-searched elements */ + if(i < end) { + CORRADE_INTERNAL_DEBUG_ASSERT(i + 16 > end); + i = end - 16; + return findCharacterSingleVectorUnaligned(Cpu::Simd128, i, vn1); + } + + return {}; + }; +} +#endif + +CORRADE_UTILITY_CPU_MAYBE_UNUSED typename std::decay::type stringFindCharacterImplementation(CORRADE_CPU_DECLARE(Cpu::Scalar)) { + return [](const char* const data, const std::size_t size, const char character) -> const char* { + /* Yet again I'm not sure if null pointers are allowed and cppreference + says nothing about that, so this might need to get patched */ return static_cast(std::memchr(data, character, size)); + }; } -inline const char* findLast(const char* const data, const std::size_t size, const char character) { +} + +#ifdef CORRADE_TARGET_X86 +CORRADE_UTILITY_CPU_DISPATCHER(stringFindCharacterImplementation, Cpu::Bmi1) +#else +CORRADE_UTILITY_CPU_DISPATCHER(stringFindCharacterImplementation) +#endif +CORRADE_UTILITY_CPU_DISPATCHED(stringFindCharacterImplementation, const char* CORRADE_UTILITY_CPU_DISPATCHED_DECLARATION(stringFindCharacter)(const char* data, std::size_t size, char character))({ + return stringFindCharacterImplementation(CORRADE_CPU_SELECT(Cpu::Default))(data, size, character); +}) + +const char* stringFindLastCharacter(const char* const data, const std::size_t size, const char character) { /* Linux has a memrchr() function but other OSes not. So let's just do it myself, that way I also don't need to worry about null pointers being allowed or not ... haha, well, except that if data is nullptr, @@ -181,7 +733,7 @@ inline const char* findLast(const char* const data, const std::size_t size, cons std::find_first_of() because I doubt STL implementations explicitly optimize for that case. Yes, std::string::find_first_of() probably would have that, but I'd first need to allocate to make use of that and FUCK NO. */ -inline const char* findAny(const char* const data, const std::size_t size, const char* const characters, const std::size_t characterCount) { +const char* stringFindAny(const char* const data, const std::size_t size, const char* const characters, const std::size_t characterCount) { for(const char* i = data, *end = data + size; i != end; ++i) if(std::memchr(characters, *i, characterCount)) return i; return {}; @@ -190,19 +742,19 @@ inline const char* findAny(const char* const data, const std::size_t size, const /* Variants of the above. Not sure if those even have any vaguely corresponding C lib API. Probably not. */ -inline const char* findLastAny(const char* const data, const std::size_t size, const char* const characters, const std::size_t characterCount) { +const char* stringFindLastAny(const char* const data, const std::size_t size, const char* const characters, const std::size_t characterCount) { for(const char* i = data + size; i != data; --i) if(std::memchr(characters, *(i - 1), characterCount)) return i - 1; return {}; } -inline const char* findNotAny(const char* const data, const std::size_t size, const char* const characters, std::size_t characterCount) { +const char* stringFindNotAny(const char* const data, const std::size_t size, const char* const characters, const std::size_t characterCount) { for(const char* i = data, *end = data + size; i != end; ++i) if(!std::memchr(characters, *i, characterCount)) return i; return {}; } -inline const char* findLastNotAny(const char* const data, const size_t size, const char* const characters, std::size_t characterCount) { +const char* stringFindLastNotAny(const char* const data, const std::size_t size, const char* const characters, const std::size_t characterCount) { for(const char* i = data + size; i != data; --i) if(!std::memchr(characters, *(i - 1), characterCount)) return i - 1; return {}; @@ -218,7 +770,7 @@ template Array> BasicStringView::splitOnAnyWithou T* const end = _data + size(); while(oldpos < end) { - if(T* const pos = const_cast(Containers::findAny(oldpos, end - oldpos, characters, characterCount))) { + if(T* const pos = const_cast(Implementation::stringFindAny(oldpos, end - oldpos, characters, characterCount))) { if(pos != oldpos) arrayAppend(parts, slice(oldpos, pos)); oldpos = pos + 1; @@ -408,10 +960,6 @@ template BasicStringView BasicStringView::exceptSuffix(const Stri return exceptSuffix(suffix.size()); } -template BasicStringView BasicStringView::trimmed(const StringView characters) const { - return trimmedPrefix(characters).trimmedSuffix(characters); -} - template BasicStringView BasicStringView::trimmed() const { #if !defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_CLANG_CL) || _MSC_VER >= 1930 /* MSVC 2022 works */ return trimmed(Whitespace); @@ -421,12 +969,6 @@ template BasicStringView BasicStringView::trimmed() const { #endif } -template BasicStringView BasicStringView::trimmedPrefix(const StringView characters) const { - const std::size_t size = this->size(); - T* const found = const_cast(findNotAny(_data, size, characters._data, characters.size())); - return suffix(found ? found : _data + size); -} - template BasicStringView BasicStringView::trimmedPrefix() const { #if !defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_CLANG_CL) || _MSC_VER >= 1930 /* MSVC 2022 works */ return trimmedPrefix(Whitespace); @@ -436,11 +978,6 @@ template BasicStringView BasicStringView::trimmedPrefix() const { #endif } -template BasicStringView BasicStringView::trimmedSuffix(const StringView characters) const { - T* const found = const_cast(findLastNotAny(_data, size(), characters._data, characters.size())); - return prefix(found ? found + 1 : _data); -} - template BasicStringView BasicStringView::trimmedSuffix() const { #if !defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_CLANG_CL) || _MSC_VER >= 1930 /* MSVC 2022 works */ return trimmedSuffix(Whitespace); @@ -450,82 +987,6 @@ template BasicStringView BasicStringView::trimmedSuffix() const { #endif } -template BasicStringView BasicStringView::findOr(const StringView substring, T* const fail) const { - /* Cache the getters to speed up debug builds */ - const std::size_t substringSize = substring.size(); - if(const char* const found = Containers::find(_data, size(), substring._data, substringSize)) - return slice(const_cast(found), const_cast(found + substringSize)); - - /* Using an internal assert-less constructor, the public constructor - asserts would be redundant. Since it's a zero-sized view, it doesn't - really make sense to try to preserve any flags. */ - return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; -} - -template BasicStringView BasicStringView::findOr(const char character, T* const fail) const { - if(const char* const found = Containers::find(_data, size(), character)) - return slice(const_cast(found), const_cast(found + 1)); - - /* Using an internal assert-less constructor, the public constructor - asserts would be redundant. Since it's a zero-sized view, it doesn't - really make sense to try to preserve any flags. */ - return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; -} - -template BasicStringView BasicStringView::findLastOr(const StringView substring, T* const fail) const { - /* Cache the getters to speed up debug builds */ - const std::size_t substringSize = substring.size(); - if(const char* const found = Containers::findLast(_data, size(), substring._data, substringSize)) - return slice(const_cast(found), const_cast(found + substringSize)); - - /* Using an internal assert-less constructor, the public constructor - asserts would be redundant. Since it's a zero-sized view, it doesn't - really make sense to try to preserve any flags. */ - return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; -} - -template BasicStringView BasicStringView::findLastOr(const char character, T* const fail) const { - if(const char* const found = Containers::findLast(_data, size(), character)) - return slice(const_cast(found), const_cast(found + 1)); - - /* Using an internal assert-less constructor, the public constructor - asserts would be redundant. Since it's a zero-sized view, it doesn't - really make sense to try to preserve any flags. */ - return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; -} - -template bool BasicStringView::contains(const StringView substring) const { - return Containers::find(_data, size(), substring._data, substring.size()); -} - -template bool BasicStringView::contains(const char character) const { - return Containers::find(_data, size(), character); -} - -template BasicStringView BasicStringView::findAnyOr(const StringView characters, T* const fail) const { - if(const char* const found = Containers::findAny(_data, size(), characters._data, characters.size())) - return slice(const_cast(found), const_cast(found + 1)); - - /* Using an internal assert-less constructor, the public constructor - asserts would be redundant. Since it's a zero-sized view, it doesn't - really make sense to try to preserve any flags. */ - return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; -} - -template BasicStringView BasicStringView::findLastAnyOr(const StringView characters, T* const fail) const { - if(const char* const found = Containers::findLastAny(_data, size(), characters._data, characters.size())) - return slice(const_cast(found), const_cast(found + 1)); - - /* Using an internal assert-less constructor, the public constructor - asserts would be redundant. Since it's a zero-sized view, it doesn't - really make sense to try to preserve any flags. */ - return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; -} - -template bool BasicStringView::containsAny(const StringView characters) const { - return Containers::findAny(_data, size(), characters._data, characters.size()); -} - #ifndef DOXYGEN_GENERATING_OUTPUT template class /* GCC needs the export macro on the class definition (and here it warns diff --git a/src/Corrade/Containers/StringView.h b/src/Corrade/Containers/StringView.h index e2347bf1a..10d145345 100644 --- a/src/Corrade/Containers/StringView.h +++ b/src/Corrade/Containers/StringView.h @@ -35,6 +35,7 @@ #include #include +#include "Corrade/Corrade.h" #include "Corrade/Containers/Containers.h" #include "Corrade/Containers/EnumSet.h" #include "Corrade/Utility/DebugAssert.h" @@ -269,7 +270,15 @@ when you deal with unordered containers. @experimental */ /* All member functions are const because the view doesn't own the data */ -template class CORRADE_UTILITY_EXPORT BasicStringView { +template class +#ifndef CORRADE_TARGET_MSVC +/* If it's here, MSVC complains that the out-of-class inline functions have a + definition while being dllimport'd. If I remove it, GCC then complains that + the export in StringView.cpp is ignored as the type is already defined, + proceeding with a linker error. */ +CORRADE_UTILITY_EXPORT +#endif +BasicStringView { public: /** * @brief Default constructor @@ -796,7 +805,9 @@ template class CORRADE_UTILITY_EXPORT BasicStringView { * @see @ref trimmed() const, @ref trimmedPrefix(StringView) const, * @ref trimmedSuffix(StringView) const */ - BasicStringView trimmed(StringView characters) const; + BasicStringView trimmed(StringView characters) const { + return trimmedPrefix(characters).trimmedSuffix(characters); + } /** * @brief View with whitespace trimmed from prefix and suffix @@ -1254,6 +1265,109 @@ template constexpr BasicStringView BasicStringView::slice(const s nullptr}; } +namespace Implementation { + +/* Making naming unique in order to prepare for these being function pointers + (that can't be overloaded) */ +CORRADE_UTILITY_EXPORT const char* stringFindString(const char* data, std::size_t size, const char* substring, std::size_t substringSize); +CORRADE_UTILITY_EXPORT const char* stringFindLastString(const char* data, std::size_t size, const char* substring, std::size_t substringSize); +CORRADE_UTILITY_EXPORT extern const char* CORRADE_UTILITY_CPU_DISPATCHED_DECLARATION(stringFindCharacter)(const char* data, std::size_t size, char character); +CORRADE_UTILITY_CPU_DISPATCHER_DECLARATION(stringFindCharacter) +CORRADE_UTILITY_EXPORT const char* stringFindLastCharacter(const char* data, std::size_t size, char character); +CORRADE_UTILITY_EXPORT const char* stringFindAny(const char* data, std::size_t size, const char* characters, std::size_t characterCount); +CORRADE_UTILITY_EXPORT const char* stringFindLastAny(const char* data, std::size_t size, const char* characters, std::size_t characterCount); +CORRADE_UTILITY_EXPORT const char* stringFindNotAny(const char* data, std::size_t size, const char* characters, std::size_t characterCount); +CORRADE_UTILITY_EXPORT const char* stringFindLastNotAny(const char* data, std::size_t size, const char* characters, std::size_t characterCount); + +} + +template inline BasicStringView BasicStringView::trimmedPrefix(const StringView characters) const { + const std::size_t size = this->size(); + T* const found = const_cast(Implementation::stringFindNotAny(_data, size, characters._data, characters.size())); + return suffix(found ? found : _data + size); +} + +template inline BasicStringView BasicStringView::trimmedSuffix(const StringView characters) const { + T* const found = const_cast(Implementation::stringFindLastNotAny(_data, size(), characters._data, characters.size())); + return prefix(found ? found + 1 : _data); +} + +template inline BasicStringView BasicStringView::findOr(const StringView substring, T* const fail) const { + /* Cache the getters to speed up debug builds */ + const std::size_t substringSize = substring.size(); + if(const char* const found = Implementation::stringFindString(_data, size(), substring._data, substringSize)) + return slice(const_cast(found), const_cast(found + substringSize)); + + /* Using an internal assert-less constructor, the public constructor + asserts would be redundant. Since it's a zero-sized view, it doesn't + really make sense to try to preserve any flags. */ + return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; +} + +template inline BasicStringView BasicStringView::findOr(const char character, T* const fail) const { + if(const char* const found = Implementation::stringFindCharacter(_data, size(), character)) + return slice(const_cast(found), const_cast(found + 1)); + + /* Using an internal assert-less constructor, the public constructor + asserts would be redundant. Since it's a zero-sized view, it doesn't + really make sense to try to preserve any flags. */ + return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; +} + +template inline BasicStringView BasicStringView::findLastOr(const StringView substring, T* const fail) const { + /* Cache the getters to speed up debug builds */ + const std::size_t substringSize = substring.size(); + if(const char* const found = Implementation::stringFindLastString(_data, size(), substring._data, substringSize)) + return slice(const_cast(found), const_cast(found + substringSize)); + + /* Using an internal assert-less constructor, the public constructor + asserts would be redundant. Since it's a zero-sized view, it doesn't + really make sense to try to preserve any flags. */ + return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; +} + +template inline BasicStringView BasicStringView::findLastOr(const char character, T* const fail) const { + if(const char* const found = Implementation::stringFindLastCharacter(_data, size(), character)) + return slice(const_cast(found), const_cast(found + 1)); + + /* Using an internal assert-less constructor, the public constructor + asserts would be redundant. Since it's a zero-sized view, it doesn't + really make sense to try to preserve any flags. */ + return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; +} + +template inline bool BasicStringView::contains(const StringView substring) const { + return Implementation::stringFindString(_data, size(), substring._data, substring.size()); +} + +template inline bool BasicStringView::contains(const char character) const { + return Implementation::stringFindCharacter(_data, size(), character); +} + +template inline BasicStringView BasicStringView::findAnyOr(const StringView characters, T* const fail) const { + if(const char* const found = Implementation::stringFindAny(_data, size(), characters._data, characters.size())) + return slice(const_cast(found), const_cast(found + 1)); + + /* Using an internal assert-less constructor, the public constructor + asserts would be redundant. Since it's a zero-sized view, it doesn't + really make sense to try to preserve any flags. */ + return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; +} + +template inline BasicStringView BasicStringView::findLastAnyOr(const StringView characters, T* const fail) const { + if(const char* const found = Implementation::stringFindLastAny(_data, size(), characters._data, characters.size())) + return slice(const_cast(found), const_cast(found + 1)); + + /* Using an internal assert-less constructor, the public constructor + asserts would be redundant. Since it's a zero-sized view, it doesn't + really make sense to try to preserve any flags. */ + return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; +} + +template inline bool BasicStringView::containsAny(const StringView characters) const { + return Implementation::stringFindAny(_data, size(), characters._data, characters.size()); +} + }} #endif diff --git a/src/Corrade/Containers/Test/CMakeLists.txt b/src/Corrade/Containers/Test/CMakeLists.txt index 1b9b09d85..07eded83d 100644 --- a/src/Corrade/Containers/Test/CMakeLists.txt +++ b/src/Corrade/Containers/Test/CMakeLists.txt @@ -28,13 +28,28 @@ # property that would have to be set on each target separately. set(CMAKE_FOLDER "Corrade/Containers/Test") +if(CORRADE_TARGET_EMSCRIPTEN OR CORRADE_TARGET_ANDROID) + set(CONTAINERS_TEST_DIR ".") +else() + set(CONTAINERS_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +endif() + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/configure.h.cmake + ${CMAKE_CURRENT_BINARY_DIR}/configure.h) + +# In all following corrade_add_test() macros, if a test wants to use +# CorradeUtilityTestLib, it has to link to CorradeTestSuiteTestLib instead. +# Otherwise the implicitly linked CorradeTestSuite would drag in CorradeUtility +# in addition to CorradeUtilityTestLib, leading to ODR violations and making +# ASan builds fail. + corrade_add_test(ContainersAnyReferenceTest AnyReferenceTest.cpp) corrade_add_test(ContainersArrayTest ArrayTest.cpp) -corrade_add_test(ContainersArrayTupleTest ArrayTupleTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(ContainersArrayTupleTest ArrayTupleTest.cpp LIBRARIES CorradeTestSuiteTestLib) corrade_add_test(ContainersArrayViewTest ArrayViewTest.cpp) corrade_add_test(ContainersArrayViewStlTest ArrayViewStlTest.cpp) corrade_add_test(ContainersBigEnumSetTest BigEnumSetTest.cpp) -corrade_add_test(ContainersBitArrayTest BitArrayTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(ContainersBitArrayTest BitArrayTest.cpp LIBRARIES CorradeTestSuiteTestLib) corrade_add_test(ContainersBitArrayViewTest BitArrayViewTest.cpp) corrade_add_test(ContainersEnumSetTest EnumSetTest.cpp) @@ -45,9 +60,9 @@ if(CORRADE_TARGET_EMSCRIPTEN) set_property(TARGET ContainersGrowableArrayTest APPEND_STRING PROPERTY LINK_FLAGS " -s ALLOW_MEMORY_GROWTH=1") endif() corrade_add_test(ContainersGrowableArraySa___FailTest GrowableArraySanitizerFailTest.cpp) -# Not matching with the -f, as MSVC might use either /f or -f. Source: -# https://devblogs.microsoft.com/cppblog/addresssanitizer-asan-for-windows-with-msvc/ -if(CMAKE_CXX_FLAGS MATCHES "fsanitize=address") +# While Clang and GCC use -fsanitize=whatever, MSVC allows also /fsanitize=, +# so catch both. +if(CMAKE_CXX_FLAGS MATCHES "[-/]fsanitize=address") set_tests_properties(ContainersGrowableArraySa___FailTest PROPERTIES PASS_REGULAR_EXPRESSION "AddressSanitizer: container-overflow") endif() @@ -69,9 +84,15 @@ corrade_add_test(ContainersStaticArrayViewTest StaticArrayViewTest.cpp) corrade_add_test(ContainersStaticArrayViewStlTest StaticArrayViewStlTest.cpp) corrade_add_test(ContainersStridedArrayViewTest StridedArrayViewTest.cpp) corrade_add_test(ContainersStridedArrayViewStlTest StridedArrayViewStlTest.cpp) -corrade_add_test(ContainersStringTest StringTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(ContainersStringTest StringTest.cpp LIBRARIES CorradeTestSuiteTestLib) corrade_add_test(ContainersStringStlTest StringStlTest.cpp) -corrade_add_test(ContainersStringViewTest StringViewTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(ContainersStringViewTest StringViewTest.cpp LIBRARIES CorradeTestSuiteTestLib) + +corrade_add_test(ContainersStringViewBenchmark StringViewBenchmark.cpp + LIBRARIES CorradeTestSuiteTestLib + FILES StringTestFiles/lorem-ipsum.txt) +target_include_directories(ContainersStringViewBenchmark PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + corrade_add_test(ContainersTripleTest TripleTest.cpp) corrade_add_test(ContainersTripleStlTest TripleStlTest.cpp) diff --git a/src/Corrade/Containers/Test/StringTestFiles/lorem-ipsum.txt b/src/Corrade/Containers/Test/StringTestFiles/lorem-ipsum.txt new file mode 100644 index 000000000..184415cd1 --- /dev/null +++ b/src/Corrade/Containers/Test/StringTestFiles/lorem-ipsum.txt @@ -0,0 +1,9 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin suscipit pharetra elit, vel ornare dui sollicitudin vel. Sed bibendum erat ex, at gravida risus rhoncus a. Sed non placerat ipsum, vitae mattis metus. Duis libero nisi, ullamcorper euismod mauris sit amet, vulputate mollis leo. Vivamus rhoncus ante et nunc lobortis lobortis. Donec elementum felis id lorem volutpat, sed varius lectus blandit. Ut sit amet elit et diam eleifend tincidunt. Vivamus rutrum consequat euismod. Nullam pretium felis eget arcu tincidunt congue. Fusce at suscipit tellus. Sed efficitur leo at ligula tempus, eget dignissim ex lobortis. + + Maecenas sit amet ligula metus. Phasellus non pretium felis. Suspendisse at metus in magna viverra sollicitudin. Etiam felis lectus, facilisis ac malesuada non, laoreet sed arcu. Nulla congue nulla justo, vitae elementum massa mollis eu. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Aliquam commodo sapien enim, eleifend tempor leo feugiat a. Integer tincidunt, ligula at vulputate posuere, nulla diam sollicitudin augue, vitae suscipit lectus lorem nec risus. Duis ex metus, elementum sed libero scelerisque, faucibus sollicitudin elit. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Cras ac ultrices tortor, in fringilla augue. In magna nibh, molestie id lacinia ut, hendrerit sit amet lacus. + + Pellentesque ac risus sed ligula condimentum consectetur. Ut eu dolor faucibus justo consectetur imperdiet nec id risus. Etiam sodales blandit est, et consectetur ex rutrum ut. Nullam at dignissim lorem. Cras non neque a sem malesuada tincidunt et nec elit. Vivamus dignissim elit eu auctor eleifend. Donec convallis quam eu aliquet gravida. Curabitur porttitor ipsum lectus, quis varius ex vulputate ut. + + Morbi a mattis nunc. Mauris suscipit orci eget nibh tincidunt aliquet. Praesent maximus ullamcorper ligula sed pellentesque. In hac habitasse platea dictumst. Suspendisse at purus nisl. Curabitur quis odio est. Ut tempor nunc quis magna euismod, vitae interdum massa faucibus. Ut in porta mauris, rhoncus hendrerit mi. Quisque orci lacus, malesuada vel augue eget, ultricies pretium nisi. Suspendisse metus ex, tristique ac facilisis at, bibendum vel eros. Integer eleifend nibh a diam malesuada, in lacinia leo condimentum. Duis dignissim rutrum odio, nec feugiat sapien elementum nec. Donec quis augue elit. Cras laoreet, sapien sed rhoncus dapibus, ligula mauris placerat enim, sed mattis sem ipsum sit amet mi. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed imperdiet purus urna, volutpat dictum ante interdum in. + + Mauris a massa pharetra, gravida ligula a, fringilla orci. Praesent tempor pretium pretium. Mauris tincidunt pellentesque maximus. Quisque eget mauris diam. In nec porttitor felis, et finibus mauris. Donec quis quam risus. Vivamus blandit ornare massa, in vulputate purus tincidunt a. Nam at erat non lorem iaculis faucibus. Nulla eu magna odio. Mauris dignissim tempor velit, ut pretium eros dignissim id. Duis fermentum viverra lectus ut dapibus. Nullam id turpis efficitur velit iaculis varius vitae vel nunc. Morbi urna velit, consequat mollis dui nec, vestibulum dictum tellus. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Pellentesque in pulvinar lacus, et vulputate turpis. diff --git a/src/Corrade/Containers/Test/StringViewBenchmark.cpp b/src/Corrade/Containers/Test/StringViewBenchmark.cpp new file mode 100644 index 000000000..0ce2a15a4 --- /dev/null +++ b/src/Corrade/Containers/Test/StringViewBenchmark.cpp @@ -0,0 +1,607 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include +#include + +#include "Corrade/Cpu.h" +#include "Corrade/Containers/Optional.h" +#include "Corrade/Containers/ArrayView.h" +#include "Corrade/Containers/StringView.h" +#include "Corrade/Containers/StringStl.h" +#include "Corrade/TestSuite/Tester.h" +#include "Corrade/Utility/Format.h" +#include "Corrade/Utility/Math.h" +#include "Corrade/Utility/Path.h" +#include "Corrade/Utility/Test/cpuVariantHelpers.h" + +#include "configure.h" + +namespace Corrade { namespace Containers { namespace Test { namespace { + +struct StringViewBenchmark: TestSuite::Tester { + explicit StringViewBenchmark(); + + void captureImplementations(); + void restoreImplementations(); + + /* The "Common" variants test rather the call / preamble / postamble + overhead, while the "Rare" variants test the actual vectorized + implementation perf */ + + void findCharacterCommon(); + void findCharacterCommonNaive(); + void findCharacterCommonMemchr(); + void findCharacterCommonStlString(); + + void findCharacterCommonSmall(); + void findCharacterCommonSmallMemchr(); + /* No std::string variant as the overhead from slicing would make this + useless (and no, find() has no end position) */ + + void findCharacterRare(); + void findCharacterRareNaive(); + void findCharacterRareMemchr(); + void findCharacterRareStlString(); + + void findLastCharacterCommon(); + void findLastCharacterCommonNaive(); + void findLastCharacterCommonMemrchr(); + void findLastCharacterCommonStlString(); + + void findLastCharacterCommonSmall(); + void findLastCharacterCommonSmallMemrchr(); + /* No std::string variant as the overhead from slicing would make this + useless (and no, rfind() has no end position) */ + + void findLastCharacterRare(); + void findLastCharacterRareNaive(); + void findLastCharacterRareMemrchr(); + void findLastCharacterRareStlString(); + + private: + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + decltype(Implementation::stringFindCharacter) findCharacterImplementation; + #endif +}; + +using namespace Containers::Literals; + +const struct { + Cpu::Features features; +} FindCharacterData[]{ + {Cpu::Scalar}, + #if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_BMI1) + {Cpu::Sse2|Cpu::Bmi1}, + #endif + #if defined(CORRADE_ENABLE_AVX2) && defined(CORRADE_ENABLE_BMI1) + {Cpu::Avx2|Cpu::Bmi1}, + #endif + #ifdef CORRADE_ENABLE_NEON + {Cpu::Neon}, + #endif + #ifdef CORRADE_ENABLE_SIMD128 + {Cpu::Simd128}, + #endif +}; + +const struct { + Cpu::Features features; + std::size_t size; +} FindCharacterSmallData[]{ + {Cpu::Scalar, 15}, + #if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_BMI1) + {Cpu::Sse2|Cpu::Bmi1, 15}, + #endif + #if defined(CORRADE_ENABLE_AVX2) && defined(CORRADE_ENABLE_BMI1) + {Cpu::Avx2|Cpu::Bmi1, 15}, + {Cpu::Avx2|Cpu::Bmi1, 31}, + #endif + #ifdef CORRADE_ENABLE_NEON + {Cpu::Neon, 15}, + #endif + #ifdef CORRADE_ENABLE_SIMD128 + {Cpu::Simd128, 15}, + #endif +}; + +StringViewBenchmark::StringViewBenchmark() { + addInstancedBenchmarks({&StringViewBenchmark::findCharacterCommon}, 100, + Utility::Test::cpuVariantCount(FindCharacterData), + &StringViewBenchmark::captureImplementations, + &StringViewBenchmark::restoreImplementations); + + addBenchmarks({&StringViewBenchmark::findCharacterCommonNaive, + &StringViewBenchmark::findCharacterCommonMemchr, + &StringViewBenchmark::findCharacterCommonStlString}, 100); + + addInstancedBenchmarks({&StringViewBenchmark::findCharacterCommonSmall}, 100, + Utility::Test::cpuVariantCount(FindCharacterSmallData), + &StringViewBenchmark::captureImplementations, + &StringViewBenchmark::restoreImplementations); + + addBenchmarks({&StringViewBenchmark::findCharacterCommonSmallMemchr}, 100); + + addInstancedBenchmarks({&StringViewBenchmark::findCharacterRare}, 100, + Utility::Test::cpuVariantCount(FindCharacterData), + &StringViewBenchmark::captureImplementations, + &StringViewBenchmark::restoreImplementations); + + addBenchmarks({&StringViewBenchmark::findCharacterRareNaive, + &StringViewBenchmark::findCharacterRareMemchr, + &StringViewBenchmark::findCharacterRareStlString, + + &StringViewBenchmark::findLastCharacterCommon, + &StringViewBenchmark::findLastCharacterCommonNaive, + &StringViewBenchmark::findLastCharacterCommonMemrchr, + &StringViewBenchmark::findLastCharacterCommonStlString, + + &StringViewBenchmark::findLastCharacterCommonSmall, + &StringViewBenchmark::findLastCharacterCommonSmallMemrchr, + + &StringViewBenchmark::findLastCharacterRare, + &StringViewBenchmark::findLastCharacterRareNaive, + &StringViewBenchmark::findLastCharacterRareMemrchr, + &StringViewBenchmark::findLastCharacterRareStlString}, 100); +} + +void StringViewBenchmark::captureImplementations() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + findCharacterImplementation = Implementation::stringFindCharacter; + #endif +} + +void StringViewBenchmark::restoreImplementations() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + Implementation::stringFindCharacter = findCharacterImplementation; + #endif +} + +constexpr std::size_t CommonCharacterCount = 500; +constexpr std::size_t RareCharacterCount = 90; +constexpr std::size_t CharacterRepeats = 100; + +void StringViewBenchmark::findCharacterCommon() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + auto&& data = FindCharacterData[testCaseInstanceId()]; + Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); + #else + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterData); + #endif + setTestCaseDescription(Utility::Test::cpuVariantName(data)); + + if(!Utility::Test::isCpuVariantSupported(data)) + CORRADE_SKIP("CPU features not supported"); + + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + StringView a = *text; + while(StringView found = a.find(' ')) { + ++count; + a = a.suffix(found.end()); + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterCommonNaive() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + const char* a = text->data(); + for(;;) { + const char* found = nullptr; + for(const char* i = a; i != text->end(); ++i) { + if(*i == ' ') { + found = i; + break; + } + } + if(!found) break; + + ++count; + a = found + 1; + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterCommonMemchr() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + const char* a = text->data(); + while(const char* found = static_cast(std::memchr(a, ' ', text->end() - a))) { + ++count; + a = found + 1; + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterCommonStlString() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + std::string a = *text; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t pos = 0; + std::size_t found; + while((found = a.find(' ', pos)) != std::string::npos) { + ++count; + pos = found + 1; + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterCommonSmall() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + auto&& data = FindCharacterSmallData[testCaseInstanceId()]; + Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); + #else + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterSmallData); + #endif + setTestCaseDescription(Utility::format("{}, {} bytes", Utility::Test::cpuVariantName(data), data.size)); + + if(!Utility::Test::isCpuVariantSupported(data)) + CORRADE_SKIP("CPU features not supported"); + + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + StringView a = *text; + while(StringView found = a.prefix(Utility::min(data.size, a.size())).find(' ')) { + ++count; + a = a.suffix(found.end()); + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterCommonSmallMemchr() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + const char* a = text->data(); + while(const char* found = static_cast(std::memchr(a, ' ', Utility::min(std::ptrdiff_t{15}, text->end() - a)))) { + ++count; + a = found + 1; + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterRare() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + auto&& data = FindCharacterData[testCaseInstanceId()]; + Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); + #else + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterData); + #endif + setTestCaseDescription(Utility::Test::cpuVariantName(data)); + + if(!Utility::Test::isCpuVariantSupported(data)) + CORRADE_SKIP("CPU features not supported"); + + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + *text = *text*10; + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + StringView a = *text; + while(StringView found = a.find('\n')) { + ++count; + a = a.suffix(found.end()); + } + } + + CORRADE_COMPARE(count, RareCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterRareNaive() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + *text = *text*10; + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + const char* a = text->data(); + for(;;) { + const char* found = nullptr; + for(const char* i = a; i != text->end(); ++i) { + if(*i == '\n') { + found = i; + break; + } + } + if(!found) break; + + ++count; + a = found + 1; + } + } + + CORRADE_COMPARE(count, RareCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterRareMemchr() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + *text = *text*10; + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + const char* a = text->data(); + while(const char* found = static_cast(std::memchr(a, '\n', text->end() - a))) { + ++count; + a = found + 1; + } + } + + CORRADE_COMPARE(count, RareCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterRareStlString() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + *text = *text*10; + + std::size_t count = 0; + std::string a = *text; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t pos = 0; + std::size_t found; + while((found = a.find('\n', pos)) != std::string::npos) { + ++count; + pos = found + 1; + } + } + + CORRADE_COMPARE(count, RareCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findLastCharacterCommon() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + StringView a = *text; + while(StringView found = a.findLast(' ')) { + ++count; + a = a.prefix(found.begin()); + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findLastCharacterCommonNaive() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t end = text->size(); + for(;;) { + const char* found = nullptr; + for(const char* i = text->begin() + end; i != text->begin(); --i) { + if(*(i - 1) == ' ') { + found = i - 1; + break; + } + } + if(!found) break; + + ++count; + end = found - text->begin(); + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findLastCharacterCommonMemrchr() { + #if !defined(__GLIBC__) && !defined(__BIONIC__) && !defined(CORRADE_TARGET_EMSCRIPTEN) + CORRADE_SKIP("memrchr() not available"); + #else + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t end = text->size(); + while(const char* found = static_cast(memrchr(text->begin(), ' ', end))) { + ++count; + end = found - text->begin(); + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); + #endif +} + +void StringViewBenchmark::findLastCharacterCommonStlString() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + std::string a = *text; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t end = text->size(); + std::size_t found; + while((found = a.rfind(' ', end)) != std::string::npos) { + ++count; + end = found - 1; + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findLastCharacterCommonSmall() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + StringView a = *text; + /** @todo use suffix() once it takes suffix size */ + while(StringView found = a.exceptPrefix(Utility::max(std::ptrdiff_t{0}, std::ptrdiff_t(a.size()) - 15)).findLast(' ')) { + ++count; + a = a.prefix(found.begin()); + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findLastCharacterCommonSmallMemrchr() { + #if !defined(__GLIBC__) && !defined(__BIONIC__) && !defined(CORRADE_TARGET_EMSCRIPTEN) + CORRADE_SKIP("memrchr() not available"); + #else + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t end = text->size(); + while(const char* found = static_cast(memrchr(text->begin() + Utility::max(std::ptrdiff_t{0}, std::ptrdiff_t(end) - 15), ' ', end - Utility::max(std::ptrdiff_t{0}, std::ptrdiff_t(end) - 15)))) { + ++count; + end = found - text->begin(); + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); + #endif +} + +void StringViewBenchmark::findLastCharacterRare() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + *text = *text*10; + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + StringView a = *text; + while(StringView found = a.findLast('\n')) { + ++count; + a = a.prefix(found.begin()); + } + } + + CORRADE_COMPARE(count, RareCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findLastCharacterRareNaive() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + *text = *text*10; + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t end = text->size(); + for(;;) { + const char* found = nullptr; + for(const char* i = text->begin() + end; i != text->begin(); --i) { + if(*(i - 1) == '\n') { + found = i - 1; + break; + } + } + if(!found) break; + + ++count; + end = found - text->begin(); + } + } + + CORRADE_COMPARE(count, RareCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findLastCharacterRareMemrchr() { + #if !defined(__GLIBC__) && !defined(__BIONIC__) && !defined(CORRADE_TARGET_EMSCRIPTEN) + CORRADE_SKIP("memrchr() not available"); + #else + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + *text = *text*10; + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t end = text->size(); + while(const char* found = static_cast(memrchr(text->begin(), '\n', end))) { + ++count; + end = found - text->begin(); + } + } + + CORRADE_COMPARE(count, RareCharacterCount*CharacterRepeats); + #endif +} + +void StringViewBenchmark::findLastCharacterRareStlString() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + *text = *text*10; + + std::size_t count = 0; + std::string a = *text; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t end = text->size(); + std::size_t found; + while((found = a.rfind('\n', end)) != std::string::npos) { + ++count; + end = found - 1; + } + } + + CORRADE_COMPARE(count, RareCharacterCount*CharacterRepeats); +} + +}}}} + +CORRADE_TEST_MAIN(Corrade::Containers::Test::StringViewBenchmark) diff --git a/src/Corrade/Containers/Test/StringViewTest.cpp b/src/Corrade/Containers/Test/StringViewTest.cpp index 6c4413c69..b0ee0345c 100644 --- a/src/Corrade/Containers/Test/StringViewTest.cpp +++ b/src/Corrade/Containers/Test/StringViewTest.cpp @@ -26,12 +26,16 @@ #include +#include "Corrade/Cpu.h" #include "Corrade/Containers/Array.h" #include "Corrade/Containers/StaticArray.h" #include "Corrade/Containers/StringView.h" #include "Corrade/TestSuite/Tester.h" #include "Corrade/TestSuite/Compare/Container.h" +#include "Corrade/TestSuite/Compare/Numeric.h" #include "Corrade/Utility/DebugStl.h" /** @todo remove once Debug is stream-free */ +#include "Corrade/Utility/Memory.h" +#include "Corrade/Utility/Test/cpuVariantHelpers.h" namespace { @@ -95,6 +99,9 @@ namespace Test { namespace { struct StringViewTest: TestSuite::Tester { explicit StringViewTest(); + void captureImplementations(); + void restoreImplementations(); + template void constructDefault(); void constructDefaultConstexpr(); template void construct(); @@ -169,16 +176,22 @@ struct StringViewTest: TestSuite::Tester { void trimmedNullView(); /* Tests also contains() */ - void find(); - void findMultipleOccurences(); - void findWholeString(); + void findString(); + void findStringMultipleOccurences(); + void findStringWhole(); + void findCharacter(); + void findCharacterAligned(); + void findCharacterUnaligned(); + void findCharacterUnalignedLessThanTwoVectors(); + void findCharacterUnalignedLessThanOneVector(); void findEmpty(); void findFlags(); void findOr(); - void findLast(); - void findLastMultipleOccurences(); - void findLastWholeString(); + void findLastString(); + void findLastStringMultipleOccurences(); + void findLastStringWhole(); + void findLastCharacter(); void findLastEmpty(); void findLastFlags(); void findLastOr(); @@ -196,6 +209,28 @@ struct StringViewTest: TestSuite::Tester { void debugFlag(); void debugFlags(); void debug(); + + private: + decltype(Implementation::stringFindCharacter) findCharacterImplementation; +}; + +const struct { + Cpu::Features features; + std::size_t vectorSize; +} FindCharacterData[]{ + {Cpu::Scalar, 16}, + #if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_BMI1) + {Cpu::Sse2|Cpu::Bmi1, 16}, + #endif + #if defined(CORRADE_ENABLE_AVX2) && defined(CORRADE_ENABLE_BMI1) + {Cpu::Avx2|Cpu::Bmi1, 32}, + #endif + #ifdef CORRADE_ENABLE_NEON + {Cpu::Neon, 16}, + #endif + #ifdef CORRADE_ENABLE_SIMD128 + {Cpu::Simd128, 16}, + #endif }; StringViewTest::StringViewTest() { @@ -275,16 +310,27 @@ StringViewTest::StringViewTest() { &StringViewTest::trimmedFlags, &StringViewTest::trimmedNullView, - &StringViewTest::find, - &StringViewTest::findMultipleOccurences, - &StringViewTest::findWholeString, - &StringViewTest::findEmpty, + &StringViewTest::findString, + &StringViewTest::findStringMultipleOccurences, + &StringViewTest::findStringWhole}); + + addInstancedTests({&StringViewTest::findCharacter, + &StringViewTest::findCharacterAligned, + &StringViewTest::findCharacterUnaligned, + &StringViewTest::findCharacterUnalignedLessThanTwoVectors, + &StringViewTest::findCharacterUnalignedLessThanOneVector}, + Utility::Test::cpuVariantCount(FindCharacterData), + &StringViewTest::captureImplementations, + &StringViewTest::restoreImplementations); + + addTests({&StringViewTest::findEmpty, &StringViewTest::findFlags, &StringViewTest::findOr, - &StringViewTest::findLast, - &StringViewTest::findLastMultipleOccurences, - &StringViewTest::findLastWholeString, + &StringViewTest::findLastString, + &StringViewTest::findLastStringMultipleOccurences, + &StringViewTest::findLastStringWhole, + &StringViewTest::findLastCharacter, &StringViewTest::findLastEmpty, &StringViewTest::findLastFlags, &StringViewTest::findLastOr, @@ -314,6 +360,18 @@ template<> struct NameFor { static const char* name() { return "MutableStringView"; } }; +void StringViewTest::captureImplementations() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + findCharacterImplementation = Implementation::stringFindCharacter; + #endif +} + +void StringViewTest::restoreImplementations() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + Implementation::stringFindCharacter = findCharacterImplementation; + #endif +} + template void StringViewTest::constructDefault() { setTestCaseTemplateName(NameFor::name()); @@ -1484,7 +1542,7 @@ void StringViewTest::trimmedNullView() { CORRADE_VERIFY(!StringView{nullptr}.trimmed().data()); } -void StringViewTest::find() { +void StringViewTest::findString() { StringView a = "hello cursed\0world!"_s; /* Right at the start */ @@ -1542,9 +1600,67 @@ void StringViewTest::find() { StringView found = a.find("world!\0"_s); CORRADE_VERIFY(!found.data()); CORRADE_VERIFY(found.isEmpty()); + } +} - /* Single character at the start */ +void StringViewTest::findStringMultipleOccurences() { + StringView a = "so, hello hell hello! hello"_s; + + /* Multiple occurrences */ + { + CORRADE_VERIFY(a.contains("hello")); + + StringView found = a.find("hello"); + CORRADE_COMPARE(found, "hello"); + CORRADE_COMPARE((static_cast(found.data())), a.data() + 4); + + /* First occurrence almost but not quite complete */ + } { + CORRADE_VERIFY(a.contains("hello!")); + + StringView found = a.find("hello!"); + CORRADE_COMPARE(found, "hello!"); + CORRADE_COMPARE((static_cast(found.data())), a.data() + 15); + } +} + +void StringViewTest::findStringWhole() { + StringView a = "hell"_s; + + /* Finding a substring that's the whole string should succeed */ + { + CORRADE_VERIFY(a.contains("hell")); + + StringView found = a.find("hell"); + CORRADE_COMPARE(found, "hell"); + CORRADE_COMPARE((static_cast(found.data())), a.data()); + + /* But a larger string should fail */ } { + CORRADE_VERIFY(!a.contains("hello")); + + StringView found = a.find("hello"); + CORRADE_VERIFY(!found.data()); + CORRADE_VERIFY(found.isEmpty()); + } +} + +void StringViewTest::findCharacter() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + auto&& data = FindCharacterData[testCaseInstanceId()]; + Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); + #else + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterData); + #endif + setTestCaseDescription(Utility::Test::cpuVariantName(data)); + + if(!Utility::Test::isCpuVariantSupported(data)) + CORRADE_SKIP("CPU features not supported"); + + StringView a = "hello cursed\0world!"_s; + + /* Single character at the start */ + { CORRADE_VERIFY(a.contains('h')); StringView found = a.find('h'); @@ -1583,68 +1699,228 @@ void StringViewTest::find() { StringView found = a.exceptPrefix(15).find('\0'); CORRADE_VERIFY(!found.data()); CORRADE_VERIFY(found.isEmpty()); - } -} - -void StringViewTest::findMultipleOccurences() { - StringView a = "so, hello hell hello! hello"_s; /* Multiple occurrences */ - { - CORRADE_VERIFY(a.contains("hello")); - - StringView found = a.find("hello"); - CORRADE_COMPARE(found, "hello"); - CORRADE_COMPARE((static_cast(found.data())), a.data() + 4); - - /* First occurrence almost but not quite complete */ - } { - CORRADE_VERIFY(a.contains("hello!")); - - StringView found = a.find("hello!"); - CORRADE_COMPARE(found, "hello!"); - CORRADE_COMPARE((static_cast(found.data())), a.data() + 15); - - /* Multiple character occurrences */ } { CORRADE_VERIFY(a.contains('o')); StringView found = a.find('o'); CORRADE_COMPARE(found, "o"); - CORRADE_COMPARE((static_cast(found.data())), a.data() + 1); + CORRADE_COMPARE((static_cast(found.data())), a.data() + 4); } } -void StringViewTest::findWholeString() { - StringView a = "hell"_s; - - /* Finding a substring that's the whole string should succeed */ - { - CORRADE_VERIFY(a.contains("hell")); - - StringView found = a.find("hell"); - CORRADE_COMPARE(found, "hell"); - CORRADE_COMPARE((static_cast(found.data())), a.data()); - - /* But a larger string should fail */ - } { - CORRADE_VERIFY(!a.contains("hello")); - - StringView found = a.find("hello"); - CORRADE_VERIFY(!found.data()); - CORRADE_VERIFY(found.isEmpty()); - } +void StringViewTest::findCharacterAligned() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + auto&& data = FindCharacterData[testCaseInstanceId()]; + Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); + #else + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterData); + #endif + setTestCaseDescription(Utility::Test::cpuVariantName(data)); + + if(!Utility::Test::isCpuVariantSupported(data)) + CORRADE_SKIP("CPU features not supported"); + + /* Allocating an array to not have it null-terminated or SSO'd in order to + trigger ASan if the algorithm goes OOB. Also, aligned, with 12 vectors + in total, corresponding to the code paths: + + +----+ +----+----+----+----+ +----+----+----+ + |deef| ! gg : hh :i i: jj | |k | ll | m| + +----+ +----+----+----+----+ +----+----+----+ + */ + Containers::Array a; + if(data.vectorSize == 16) + a = Utility::allocateAligned(Corrade::ValueInit, data.vectorSize*(1 + 4*2 + 3)); + else if(data.vectorSize == 32) + a = Utility::allocateAligned(Corrade::ValueInit, data.vectorSize*(1 + 4*2 + 3)); + else CORRADE_INTERNAL_ASSERT_UNREACHABLE(); + MutableStringView string = arrayView(a); + CORRADE_COMPARE_AS(string.data(), data.vectorSize, + TestSuite::Compare::Aligned); + + /* First vector is treated separately. It should pick the first found of + the two; test also the very first and very last. */ + string[0] = 'd'; + string[7] = 'e'; + string[data.vectorSize - 7] = 'e'; + string[data.vectorSize - 1] = 'f'; + CORRADE_COMPARE(string.find('d').data() - string.data(), 0); + CORRADE_COMPARE(string.find('e').data() - string.data(), 7); + CORRADE_COMPARE(string.find('f').data() - string.data(), data.vectorSize - 1); + + /* Then it's four vectors at a time. First four would be empty, second four + would have the data. Test each of the four separately. For each it + should pick the first found of the two. */ + string[data.vectorSize*5 + 3] = 'g'; + string[data.vectorSize*6 - 3] = 'g'; + string[data.vectorSize*6 + 7] = 'h'; + string[data.vectorSize*7 - 7] = 'h'; + string[data.vectorSize*7 + 0] = 'i'; + string[data.vectorSize*8 - 1] = 'i'; + string[data.vectorSize*8 + 2] = 'j'; + string[data.vectorSize*9 - 2] = 'j'; + CORRADE_COMPARE(string.find('g').data() - string.data(), data.vectorSize*5 + 3); + CORRADE_COMPARE(string.find('h').data() - string.data(), data.vectorSize*6 + 7); + CORRADE_COMPARE(string.find('i').data() - string.data(), data.vectorSize*7 + 0); + CORRADE_COMPARE(string.find('j').data() - string.data(), data.vectorSize*8 + 2); + + /* Last less-than-four vectors are again treated separately. Again, for + each it should pick the last found of the two; test also the very first + and very last of the range. */ + string[data.vectorSize* 9 + 0] = 'k'; + string[data.vectorSize*10 + 4] = 'l'; + string[data.vectorSize*11 - 4] = 'l'; + string[data.vectorSize*12 - 1] = 'm'; + CORRADE_COMPARE(string.find('k').data() - string.data(), data.vectorSize*9 + 0); + CORRADE_COMPARE(string.find('l').data() - string.data(), data.vectorSize*10 + 4); + CORRADE_COMPARE(string.find('m').data() - string.data(), data.vectorSize*12 - 1); + + /* A character that's not found should be handled properly even after all + these complex code paths */ + CORRADE_VERIFY(!string.find('n')); +} + +void StringViewTest::findCharacterUnaligned() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + auto&& data = FindCharacterData[testCaseInstanceId()]; + Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); + #else + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterData); + #endif + setTestCaseDescription(Utility::Test::cpuVariantName(data)); + + if(!Utility::Test::isCpuVariantSupported(data)) + CORRADE_SKIP("CPU features not supported"); + + /* Allocating an array to not have it null-terminated or SSO'd in order to + trigger ASan if the algorithm goes OOB. Also, aligned, but then slicing: + - the first unaligned vector having all bytes but one overlapping with + the four-at-a-time block + - there being just one four-at-a-time block (the if() branch that skips + the block was sufficiently tested in firstCharacterAligned()) + - there being just one full vector after, and the last unaligned vector + again overlapping with all but one byte with it + + +----+ +----+ + |f | | j| + +----+ +----+ + +----+----+----+----+----+ + |g : : : h|i | + +----+----+----+----+----+ + */ + Containers::Array a; + if(data.vectorSize == 16) + a = Utility::allocateAligned(Corrade::ValueInit, data.vectorSize*(1 + 4 + 2)); + else if(data.vectorSize == 32) + a = Utility::allocateAligned(Corrade::ValueInit, data.vectorSize*(1 + 4 + 2)); + else CORRADE_INTERNAL_ASSERT_UNREACHABLE(); + MutableStringView string = a.slice(data.vectorSize - 1, a.size() - (data.vectorSize - 1)); + CORRADE_COMPARE(string.size(), data.vectorSize*5 + 2); + CORRADE_COMPARE_AS(string.data(), data.vectorSize, + TestSuite::Compare::NotAligned); + + /* First byte should be handled by the initial unaligned check */ + string[0] = 'f'; + string[data.vectorSize - 1] = 'f'; + CORRADE_COMPARE(string.find('f').data() - string.data(), 0); + + /* The four-vectors-at-a-time should handle the aligned middle portion. + Test just the very first and very last of the aligned range. */ + string[data.vectorSize*0 + 1] = 'g'; + string[data.vectorSize*4 + 0] = 'h'; + CORRADE_COMPARE_AS(string.data() + 1, data.vectorSize, + TestSuite::Compare::Aligned); + CORRADE_COMPARE(string.find('g').data() - string.data(), data.vectorSize*0 + 1); + CORRADE_COMPARE(string.find('h').data() - string.data(), data.vectorSize*4 + 0); + + /* The byte right after the aligned block is handled by the "less than + four vectors" block */ + string[data.vectorSize*4 - 1] = 'i'; + CORRADE_COMPARE_AS(string.data() + data.vectorSize*4 + 1, data.vectorSize, + TestSuite::Compare::Aligned); + CORRADE_COMPARE(string.find('i').data() - string.data(), data.vectorSize*4 - 1); + + /* Last byte should be handled by the final unaligned check */ + string[string.size() - 1] = 'j'; + CORRADE_COMPARE(string.find('j').data() - string.data(), data.vectorSize*5 + 1); + + /* A character that's not found should be handled properly even after all + these complex code paths */ + CORRADE_VERIFY(!string.find('k')); +} + +void StringViewTest::findCharacterUnalignedLessThanTwoVectors() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + auto&& data = FindCharacterData[testCaseInstanceId()]; + Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); + #else + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterData); + #endif + setTestCaseDescription(Utility::Test::cpuVariantName(data)); + + if(!Utility::Test::isCpuVariantSupported(data)) + CORRADE_SKIP("CPU features not supported"); + + /* Allocating an array to not have it null-terminated or SSO'd in order to + trigger ASan if the algorithm goes OOB. Also, aligned, but then slicing + so there's just two unaligned blocks overlapping in a single byte: + + +----+ + |f | + +----+ + +----+ + |g ! + +----+ */ + Containers::Array a; + if(data.vectorSize == 16) + a = Utility::allocateAligned(Corrade::ValueInit, data.vectorSize*3); + else if(data.vectorSize == 32) + a = Utility::allocateAligned(Corrade::ValueInit, data.vectorSize*3); + else CORRADE_INTERNAL_ASSERT_UNREACHABLE(); + MutableStringView string = a.slice(2, 2 + data.vectorSize*2 - 1); + CORRADE_COMPARE_AS(string.data(), data.vectorSize, + TestSuite::Compare::NotAligned); + + /* First byte should be handled by the initial unaligned check */ + string[0] = 'f'; + CORRADE_COMPARE(string.find('f').data() - string.data(), 0); + + /* Last byte should be handled by the final unaligned check */ + string[string.size() - 1] = 'g'; + CORRADE_COMPARE(string.find('g').data() - string.data(), data.vectorSize*2 - 2); + + /* A character that's not found should be handled properly here as well */ + CORRADE_VERIFY(!string.find('h')); +} + +void StringViewTest::findCharacterUnalignedLessThanOneVector() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + auto&& data = FindCharacterData[testCaseInstanceId()]; + Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); + #else + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterData); + #endif + setTestCaseDescription(Utility::Test::cpuVariantName(data)); - StringView b = "h"_s; + if(!Utility::Test::isCpuVariantSupported(data)) + CORRADE_SKIP("CPU features not supported"); - /* Finding a single character that's the whole string should succeed too */ - { - CORRADE_VERIFY(b.contains('h')); + /* Allocating an array to not have it null-terminated or SSO'd in order to + trigger ASan if the algorithm goes OOB. Deliberately pick an unaligned + pointer even though it shouldn't matter here. It should pick the first + found of the two. */ + Containers::Array a{Corrade::ValueInit, data.vectorSize}; + MutableStringView string = a.exceptPrefix(1); + string[7] = 'f'; + string[data.vectorSize/2 + 1] = 'f'; + CORRADE_COMPARE_AS(string.data(), data.vectorSize, + TestSuite::Compare::NotAligned); + CORRADE_COMPARE(string.find('f').data() - string.data(), 7); - StringView found = b.find('h'); - CORRADE_COMPARE(found, "h"); - CORRADE_COMPARE((static_cast(found.data())), b.data()); - } + /* A character that's not found should be handled properly here as well */ + CORRADE_VERIFY(!string.find('g')); } void StringViewTest::findEmpty() { @@ -1802,9 +2078,9 @@ void StringViewTest::findOr() { } } -void StringViewTest::findLast() { - /* Mostly similar to find(), except that it doesn't check contains() (which - is internally the same algorithm as find()) */ +void StringViewTest::findLastString() { + /* Mostly similar to findString(), except that it doesn't check contains() + (which is internally the same algorithm as find()) */ StringView a = "hello cursed\0world!"_s; @@ -1849,43 +2125,12 @@ void StringViewTest::findLast() { StringView found = a.findLast("world!\0"_s); CORRADE_VERIFY(!found.data()); CORRADE_VERIFY(found.isEmpty()); - - /* Single character at the end */ - } { - StringView found = a.findLast('!'); - CORRADE_COMPARE(found, "!"); - CORRADE_COMPARE(static_cast(found.data()), a.data() + 18); - - /* Single character in the middle */ - } { - StringView found = a.findLast('c'); - CORRADE_COMPARE(found, "c"); - CORRADE_COMPARE(static_cast(found.data()), a.data() + 6); - - /* Single character at the start */ - } { - StringView found = a.findLast('h'); - CORRADE_COMPARE(found, "h"); - CORRADE_COMPARE(static_cast(found.data()), a.data()); - - /* No such character found */ - } { - StringView found = a.findLast('a'); - CORRADE_VERIFY(!found.data()); - CORRADE_VERIFY(found.isEmpty()); - - /* Should not read the null terminator character either */ - } { - /* There's a \0 in the middle, skip that */ - StringView found = a.exceptPrefix(15).findLast('\0'); - CORRADE_VERIFY(!found.data()); - CORRADE_VERIFY(found.isEmpty()); } } -void StringViewTest::findLastMultipleOccurences() { - /* Mostly similar to findMultipleOccurences(), except that it doesn't check - contains() (which is internally the same algorithm as find()) */ +void StringViewTest::findLastStringMultipleOccurences() { + /* Mostly similar to findStringMultipleOccurences(), except that it doesn't + check contains() (which is internally the same algorithm as find()) */ StringView a = "so, hello hell hello! hello hell"_s; @@ -1900,17 +2145,11 @@ void StringViewTest::findLastMultipleOccurences() { StringView found = a.findLast("hello!"); CORRADE_COMPARE(found, "hello!"); CORRADE_COMPARE((static_cast(found.data())), a.data() + 15); - - /* Multiple character occurrences */ - } { - StringView found = a.findLast('o'); - CORRADE_COMPARE(found, "o"); - CORRADE_COMPARE((static_cast(found.data())), a.data() + 26); } } -void StringViewTest::findLastWholeString() { - /* Mostly similar to findWholeString(), except that it doesn't check +void StringViewTest::findLastStringWhole() { + /* Mostly similar to findStringWhole(), except that it doesn't check contains() (which is internally the same algorithm as find()) */ StringView a = "hell"_s; @@ -1927,14 +2166,50 @@ void StringViewTest::findLastWholeString() { CORRADE_VERIFY(!found.data()); CORRADE_VERIFY(found.isEmpty()); } +} - StringView b = "h"_s; +void StringViewTest::findLastCharacter() { + /* Mostly similar to findCharacter(), except that it doesn't check + contains() (which is internally the same algorithm as find()) */ + + StringView a = "hello cursed\0world!"_s; - /* Finding a single character that's the whole string should succeed too */ + /* Single character at the end */ { - StringView found = b.findLast('h'); + StringView found = a.findLast('!'); + CORRADE_COMPARE(found, "!"); + CORRADE_COMPARE(static_cast(found.data()), a.data() + 18); + + /* Single character in the middle */ + } { + StringView found = a.findLast('c'); + CORRADE_COMPARE(found, "c"); + CORRADE_COMPARE(static_cast(found.data()), a.data() + 6); + + /* Single character at the start */ + } { + StringView found = a.findLast('h'); CORRADE_COMPARE(found, "h"); - CORRADE_COMPARE((static_cast(found.data())), b.data()); + CORRADE_COMPARE(static_cast(found.data()), a.data()); + + /* No such character found */ + } { + StringView found = a.findLast('a'); + CORRADE_VERIFY(!found.data()); + CORRADE_VERIFY(found.isEmpty()); + + /* Should not read the null terminator character either */ + } { + /* There's a \0 in the middle, skip that */ + StringView found = a.exceptPrefix(15).findLast('\0'); + CORRADE_VERIFY(!found.data()); + CORRADE_VERIFY(found.isEmpty()); + + /* Multiple character occurrences */ + } { + StringView found = a.findLast('o'); + CORRADE_COMPARE(found, "o"); + CORRADE_COMPARE((static_cast(found.data())), a.data() + 14); } } diff --git a/src/Corrade/Containers/Test/configure.h.cmake b/src/Corrade/Containers/Test/configure.h.cmake new file mode 100644 index 000000000..c5bb625fb --- /dev/null +++ b/src/Corrade/Containers/Test/configure.h.cmake @@ -0,0 +1,29 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#define CONTAINERS_TEST_DIR "${CONTAINERS_TEST_DIR}" + +// kate: hl c++ diff --git a/src/Corrade/Corrade.h b/src/Corrade/Corrade.h index e14676d2f..b9f18d5a6 100644 --- a/src/Corrade/Corrade.h +++ b/src/Corrade/Corrade.h @@ -27,7 +27,7 @@ */ /** @file - * @brief Basic definitions + * @brief Basic definitions and forward declarations for the @ref Corrade namespace */ #include "Corrade/configure.h" @@ -115,6 +115,25 @@ read/write access to global data. #define CORRADE_BUILD_MULTITHREADED #undef CORRADE_BUILD_MULTITHREADED +/** +@brief Build with runtime CPU dispatch +@m_since_latest + +Defined if the library is built with performance-critical code paths optimized +for multiple architectures (such as SSE or AVX on x86), with the best matching +variant selected at runtime based on detected CPU features. If not defined, the +library is built with just a single variant that's picked at compile time +depending on target architecture flags being passed to the compiler. + +The actual feature detection and dispatch both in the runtime and compile-time +scenario is performed by the @relativeref{Corrade,Cpu} library. See +@ref Cpu-usage-automatic-cached-dispatch for details and information about +performance tradeoffs. +@see @see @ref CORRADE_CPU_USE_IFUNC, @ref building-corrade, @ref corrade-cmake +*/ +#define CORRADE_BUILD_CPU_RUNTIME_DISPATCH +#undef CORRADE_BUILD_CPU_RUNTIME_DISPATCH + /** @brief Debug build @@ -256,7 +275,8 @@ library doesn't know about yet. @ref CORRADE_TARGET_SSSE3, @ref CORRADE_TARGET_SSE41, @ref CORRADE_TARGET_SSE42, @ref CORRADE_TARGET_AVX, @ref CORRADE_TARGET_AVX_F16C, @ref CORRADE_TARGET_AVX_FMA, - @ref CORRADE_TARGET_AVX2, @ref CORRADE_TARGET_AVX512F + @ref CORRADE_TARGET_AVX2, @ref CORRADE_TARGET_AVX512F, + @relativeref{Corrade,Cpu} */ #define CORRADE_TARGET_X86 #undef CORRADE_TARGET_X86 @@ -274,7 +294,8 @@ unclear on platforms with multi-architecture binaries. If neither nor @ref CORRADE_TARGET_WASM is defined, the platform might be either a very old pre-WebAssembly @ref CORRADE_TARGET_EMSCRIPTEN or any other that the library doesn't know about yet. -@see @ref CORRADE_TARGET_NEON +@see @ref CORRADE_TARGET_NEON, @ref CORRADE_TARGET_NEON_FMA, + @ref CORRADE_TARGET_NEON_FP16, @relativeref{Corrade,Cpu} */ #define CORRADE_TARGET_ARM #undef CORRADE_TARGET_ARM @@ -312,7 +333,7 @@ unclear on platforms with multi-architecture binaries. If neither nor @ref CORRADE_TARGET_WASM is defined, the platform might be either a very old pre-WebAssembly @ref CORRADE_TARGET_EMSCRIPTEN or any other that the library doesn't know about yet. -@see @ref CORRADE_TARGET_SIMD128 +@see @ref CORRADE_TARGET_SIMD128, @relativeref{Corrade,Cpu} */ #define CORRADE_TARGET_WASM #undef CORRADE_TARGET_WASM @@ -512,6 +533,8 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if [Streaming SIMD Extensions 2](https://en.wikipedia.org/wiki/SSE2) are enabled at compile time (`-msse2` or higher on GCC/Clang, `/arch:SSE2` or higher on MSVC). All x86-64 targets support SSE2. Implied by @ref CORRADE_TARGET_SSE3. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Sse2}, + @ref CORRADE_ENABLE_SSE2 */ #define CORRADE_TARGET_SSE2 #undef CORRADE_TARGET_SSE2 @@ -525,6 +548,8 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if at compile time (on GCC/Clang it's `-msse3` and higher, MSVC doesn't have a direct option and it's only implied by `/arch:AVX`). Superset of @ref CORRADE_TARGET_SSE2, implied by @ref CORRADE_TARGET_SSSE3. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Sse3}, + @ref CORRADE_ENABLE_SSE3 */ #define CORRADE_TARGET_SSE3 #undef CORRADE_TARGET_SSE3 @@ -534,7 +559,7 @@ direct option and it's only implied by `/arch:AVX`). Superset of @m_since_latest Defined on @ref CORRADE_TARGET_X86 "x86" if -[Supplemental Streaming SIMD Extensions 3](https://en.wikipedia.org/wiki/SSE3) +[Supplemental Streaming SIMD Extensions 3](https://en.wikipedia.org/wiki/SSSE3) are enabled at compile time (on GCC/Clang it's `-mssse3` and higher, MSVC doesn't have a direct option and it's only implied by `/arch:AVX`). Superset of @ref CORRADE_TARGET_SSE3, implied by @ref CORRADE_TARGET_SSE41. @@ -543,6 +568,8 @@ Note that certain older AMD processors have [SSE4a](https://en.wikipedia.org/wik but neither SSSE3 nor SSE4.1. Both can be however treated as a subset of SSE4.1 to a large extent, and it's recommended to use @ref CORRADE_TARGET_SSE41 to detect those. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Ssse3}, + @ref CORRADE_ENABLE_SSSE3 */ #define CORRADE_TARGET_SSSE3 #undef CORRADE_TARGET_SSSE3 @@ -561,6 +588,8 @@ Note that certain older AMD processors have [SSE4a](https://en.wikipedia.org/wik but neither SSSE3 nor SSE4.1. Both can be however treated as a subset of SSE4.1 to a large extent, and it's recommended to use @ref CORRADE_TARGET_SSE41 to detect those. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Sse41}, + @ref CORRADE_ENABLE_SSE41 */ #define CORRADE_TARGET_SSE41 #undef CORRADE_TARGET_SSE41 @@ -574,10 +603,64 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if are enabled at compile time (on GCC/Clang it's `-msse4.2` and higher, MSVC doesn't have a direct option and it's only implied by `/arch:AVX`). Superset of @ref CORRADE_TARGET_SSE41, implied by @ref CORRADE_TARGET_AVX. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Sse42}, + @ref CORRADE_ENABLE_SSE42 */ #define CORRADE_TARGET_SSE42 #undef CORRADE_TARGET_SSE42 +/** +@brief Target with POPCNT instructions +@m_since_latest + +Defined on @ref CORRADE_TARGET_X86 "x86" if +[POPCNT](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation)) +is enabled at compile time. On GCC/Clang it's `-mpopcnt` and is also implied by +`-msse4.2` and higher, MSVC doesn't have a direct option but it's assumed to +be implied by `/arch:AVX`. To avoid failures at runtime, prefer to detect its +presence with @ref Cpu::runtimeFeatures(). +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Popcnt}, + @ref CORRADE_ENABLE_POPCNT +*/ +#define CORRADE_TARGET_POPCNT +#undef CORRADE_TARGET_POPCNT + +/** +@brief Target with LZCNT instructions +@m_since_latest + +Defined on @ref CORRADE_TARGET_X86 "x86" if +[LZCNT](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation)) +is enabled at compile time (on GCC/Clang it's `-mlznct`, MSVC doesn't have a +direct option but it's assumed to be implied by `/arch:AVX2`). However note +that this instruction has encoding compatible with an earlier `BSR` instruction +which has a slightly different behavior. To avoid wrong results if it isn't +available, prefer to detect its presence with @ref Cpu::runtimeFeatures() +instead. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Lzcnt}, + @ref CORRADE_ENABLE_LZCNT +*/ +#define CORRADE_TARGET_LZCNT +#undef CORRADE_TARGET_LZCNT + +/** +@brief Target with BMI1 instructions +@m_since_latest + +Defined on @ref CORRADE_TARGET_X86 "x86" if +[BMI1](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#BMI1_(Bit_Manipulation_Instruction_Set_1)) +including the `TZCNT` instruction is enabled at compile time (on GCC/Clang it's +`-mbmi`, MSVC doesn't have a direct option but it's assumed to be implied by +`/arch:AVX2`). However note that the `TZCNT` instruction has encoding +compatible with an earlier `BSF` instruction which has a slightly different +behavior. To avoid wrong results if it isn't available, prefer to detect its +presence with @ref Cpu::runtimeFeatures() instead. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Bmi1}, + @ref CORRADE_ENABLE_BMI1 +*/ +#define CORRADE_TARGET_BMI1 +#undef CORRADE_TARGET_BMI1 + /** @brief AVX target @m_since_latest @@ -586,7 +669,9 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if [Advanced Vector Extensions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions) are enabled at compile time (`-mavx` and higher on GCC/Clang, `/arch:AVX` on MSVC). Superset of @ref CORRADE_TARGET_SSE42, implied by -@ref CORRADE_TARGET_AVX_F16C. +@ref CORRADE_TARGET_AVX2. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Avx}, + @ref CORRADE_ENABLE_AVX */ #define CORRADE_TARGET_AVX #undef CORRADE_TARGET_AVX @@ -597,13 +682,11 @@ MSVC). Superset of @ref CORRADE_TARGET_SSE42, implied by Defined on @ref CORRADE_TARGET_X86 "x86" if the [F16C instruction set](https://en.wikipedia.org/wiki/F16C) is enabled at -compile time (`-mf16c` on GCC/Clang, MSVC doesn't have a direct option and it's -only implied by `/arch:AVX2`). Superset of @ref CORRADE_TARGET_AVX, implied by -@ref CORRADE_TARGET_AVX_FMA. - -Although there's no documented relation between AVX, F16C, FMA and AVX2, -looking the history of released Intel and AMD CPUs it can be seen that all CPUs -having F16C support plain AVX as well, and all CPUs supporting FMA have F16C. +compile time. On GCC/Clang it's `-mf16c`, MSVC doesn't have a direct option but +it's assumed to be implied by `/arch:AVX2`. To avoid failures at runtime, +prefer to detect its presence with @ref Cpu::runtimeFeatures(). +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::AvxF16c}, + @ref CORRADE_ENABLE_AVX_F16C */ #define CORRADE_TARGET_AVX_F16C #undef CORRADE_TARGET_AVX_F16C @@ -613,15 +696,16 @@ having F16C support plain AVX as well, and all CPUs supporting FMA have F16C. @m_since_latest Defined on @ref CORRADE_TARGET_X86 "x86" if the -[FMA3 instruction set](https://en.wikipedia.org/wiki/FMA_instruction_set) is enabled at compile time (`-mfma` on GCC/Clang, MSVC doesn't have a direct -option and it's only implied by `/arch:AVX2`). Superset of -@ref CORRADE_TARGET_AVX_F16C, implied by @ref CORRADE_TARGET_AVX2. +[FMA3 instruction set](https://en.wikipedia.org/wiki/FMA_instruction_set) is +enabled at compile time. On GCC/Clang it's `-mfma`, MSVC doesn't have a direct +option but it's assumes to be implied by `/arch:AVX2`. To avoid failures at +runtime, prefer to detect its presence with @ref Cpu::runtimeFeatures(). The FMA4 instruction set, which used to be supported only in certain range of AMD processors and isn't anymore, is not detected, and AMD switched to FMA3 -since. Although there's no documented relation between AVX, F16C, FMA and AVX2, -looking the history of released Intel and AMD CPUs it can be seen that all CPUs -having FMA support F16C as well, and all CPUs supporting AVX2 have FMA. +since. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::AvxFma}, + @ref CORRADE_ENABLE_AVX_FMA */ #define CORRADE_TARGET_AVX_FMA #undef CORRADE_TARGET_AVX_FMA @@ -633,8 +717,10 @@ having FMA support F16C as well, and all CPUs supporting AVX2 have FMA. Defined on @ref CORRADE_TARGET_X86 "x86" if [Advanced Vector Extensions 2](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#Advanced_Vector_Extensions_2) are enabled at compile time (`-mavx2` and higher on GCC/Clang, `/arch:AVX2` on -MSVC). Superset of @ref CORRADE_TARGET_AVX_FMA, implied by +MSVC). Superset of @ref CORRADE_TARGET_AVX, implied by @ref CORRADE_TARGET_AVX512F. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Avx2}, + @ref CORRADE_ENABLE_AVX2 */ #define CORRADE_TARGET_AVX2 #undef CORRADE_TARGET_AVX2 @@ -646,6 +732,8 @@ MSVC). Superset of @ref CORRADE_TARGET_AVX_FMA, implied by Defined on @ref CORRADE_TARGET_X86 "x86" if [AVX-512](https://en.wikipedia.org/wiki/AVX-512) Foundation instructions are enabled at compile time (`-mavx512f` and higher on GCC/Clang, `/arch:AVX512` on MSVC). Superset of @ref CORRADE_TARGET_AVX2. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Avx512f}, + @ref CORRADE_ENABLE_AVX512F */ #define CORRADE_TARGET_AVX512F #undef CORRADE_TARGET_AVX512F @@ -657,7 +745,7 @@ GCC/Clang, `/arch:AVX512` on MSVC). Superset of @ref CORRADE_TARGET_AVX2. Defined on @ref CORRADE_TARGET_ARM "ARM" if [ARM NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)) instructions are enabled at compile time (`-mfpu=neon` on GCC/Clang, implicitly -supported on AArch64). Implied by @ref CORRADE_TARGET_NEON_FP16. +supported on ARM64). Implied by @ref CORRADE_TARGET_NEON_FMA. Apart from NEON, there's several other mutually incompatible ARM instruction sets. Detection for these will be added when the platforms become more @@ -669,47 +757,73 @@ sets. Detection for these will be added when the platforms become more - SVE2, which is a next-generation vector instruction set designed to be a successor to both NEON and SVE, scheduled to appear in production in around 2022 +- AMX, which is Apple's proprietary and patented instruction set available + only on their own hardware + +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Neon}, + @ref CORRADE_ENABLE_NEON */ #define CORRADE_TARGET_NEON #undef CORRADE_TARGET_NEON -/** -@brief NEON target with half-floats -@m_since_latest - -Defined on @ref CORRADE_TARGET_ARM "ARM" if NEON IEEE -[half-precision floating-point](https://en.wikipedia.org/wiki/Half-precision_floating-point_format) -support is enabled at compile time (`-mfpu=neon-fp16` or higher on GCC/Clang). -Not defined if only the ARM alternative half-float representation is available, -which trades one extra exponent value for a lack of infinity and NaN support. -Superset of @ref CORRADE_TARGET_NEON, implied by @ref CORRADE_TARGET_NEON_FMA. -*/ -#define CORRADE_TARGET_NEON_FP16 -#undef CORRADE_TARGET_NEON_FP16 - /** @brief NEON target with FMA @m_since_latest Defined on @ref CORRADE_TARGET_ARM "ARM" if NEON FMA instructions are enabled -at compile time (`-mfpu=neon-vfpv4` on GCC/Clang). Not defined if FMA is only -available for scalar code and not for NEON. Superset of +at compile time (`-mfpu=neon-vfpv4` on GCC/Clang on 32-bit ARM, implicitly +supported on ARM64). Not defined if FMA is only available for scalar code and +not for NEON. Superset of @ref CORRADE_TARGET_NEON, implied by @ref CORRADE_TARGET_NEON_FP16. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::NeonFma}, + @ref CORRADE_ENABLE_NEON_FMA */ #define CORRADE_TARGET_NEON_FMA #undef CORRADE_TARGET_NEON_FMA +/** +@brief NEON target with FP16 vector arithmetic +@m_since_latest + +Defined on @ref CORRADE_TARGET_ARM "ARM" if ARMv8.2-a NEON FP16 vector +arithmetic support is enabled at compile time (`-march=armv8.2-a+fp16` on +GCC/Clang). Superset of @ref CORRADE_TARGET_NEON_FMA. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::NeonFp16}, + @ref CORRADE_ENABLE_NEON_FP16 +*/ +#define CORRADE_TARGET_NEON_FP16 +#undef CORRADE_TARGET_NEON_FP16 + /** @brief SIMD128 target @m_since_latest Defined on @ref CORRADE_TARGET_WASM "WebAssembly" if [128-bit SIMD](https://github.com/webassembly/simd) instructions are enabled at -compile time (`-msimd128` passed to Clang). +compile time (`-msimd128` passed to Clang), and the compiler supports the +finalized version of the intrinsics, which is since Clang 13 and Emscripten +2.0.18. Emscripten SDK 2.0.13 to 2.0.17 ship with a Clang that reports as 13 +but isn't actually the final version. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Simd128}, + @ref CORRADE_ENABLE_SIMD128 */ #define CORRADE_TARGET_SIMD128 #undef CORRADE_TARGET_SIMD128 +/** +@brief GNU IFUNC is allowed to be used for runtime dispatch in the Cpu library +@m_since_latest + +Defined if the @relativeref{Corrade,Cpu} library can perform runtime dispatch +using [GNU IFUNC](https://sourceware.org/glibc/wiki/GNU_IFUNC), exposing the +@ref CORRADE_CPU_DISPATCHED_IFUNC() macro. Supported only on Linux with glibc +and on Android with API 30+. See @ref Cpu-usage-automatic-cached-dispatch for +details and information about performance tradeoffs. +@see @ref building-corrade, @ref corrade-cmake +*/ +#define CORRADE_CPU_USE_IFUNC +#undef CORRADE_CPU_USE_IFUNC + /** @brief PluginManager doesn't have dynamic plugin support on this platform @@ -751,6 +865,12 @@ the console. This is done automatically when you link to the #undef CORRADE_UTILITY_USE_ANSI_COLORS #endif +#ifndef DOXYGEN_GENERATING_OUTPUT +namespace Cpu { + class Features; +} +#endif + } #endif diff --git a/src/Corrade/Cpu.cpp b/src/Corrade/Cpu.cpp new file mode 100644 index 000000000..c782621d8 --- /dev/null +++ b/src/Corrade/Cpu.cpp @@ -0,0 +1,188 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include "Cpu.h" + +#include "Corrade/Containers/StringView.h" +#include "Corrade/Utility/Debug.h" + +/* getauxval() for ARM on Linux and Android with API level 18+ */ +#if defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) +#include + +/* sysctlbyname() for ARM on macOS / iOS */ +#elif defined(CORRADE_TARGET_ARM) && defined(CORRADE_TARGET_APPLE) +#include +#endif + +namespace Corrade { namespace Cpu { + +using namespace Containers::Literals; + +/* As the types all inherit from each other, there should be no members to + keep them zero-cost. */ +static_assert(sizeof(Cpu::Scalar) == 1, ""); +#ifdef CORRADE_TARGET_X86 +static_assert(sizeof(Cpu::Sse2) == 1, ""); +static_assert(sizeof(Cpu::Sse3) == 1, ""); +static_assert(sizeof(Cpu::Ssse3) == 1, ""); +static_assert(sizeof(Cpu::Sse41) == 1, ""); +static_assert(sizeof(Cpu::Sse42) == 1, ""); +static_assert(sizeof(Cpu::Avx) == 1, ""); +static_assert(sizeof(Cpu::AvxF16c) == 1, ""); +static_assert(sizeof(Cpu::AvxFma) == 1, ""); +static_assert(sizeof(Cpu::Avx2) == 1, ""); +static_assert(sizeof(Cpu::Avx512f) == 1, ""); +#elif defined(CORRADE_TARGET_ARM) +static_assert(sizeof(Cpu::Neon) == 1, ""); +static_assert(sizeof(Cpu::NeonFma) == 1, ""); +static_assert(sizeof(Cpu::NeonFp16) == 1, ""); +#elif defined(CORRADE_TARGET_WASM) +static_assert(sizeof(Cpu::Simd128) == 1, ""); +#endif + +/* Helper for getting macOS / iOS ARM properties. Yep, it's stringly typed. */ +#if defined(CORRADE_TARGET_ARM) && defined(CORRADE_TARGET_APPLE) +namespace { + +int appleSysctlByName(const char* name) { + int value; + std::size_t size = sizeof(value); + /* First pointer/size pair is for querying the value, second is for setting + the value. Returns 0 on success. */ + return sysctlbyname(name, &value, &size, nullptr, 0) ? 0 : value; +} + +} +#endif + +#if defined(CORRADE_TARGET_ARM) && ((defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18)) || defined(CORRADE_TARGET_APPLE)) +Features runtimeFeatures() { + /* Use getauxval() on ARM on Linux and Android */ + #if defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) + /* People say getauxval() is "extremely slow": + https://lemire.me/blog/2020/07/17/the-cost-of-runtime-dispatch/#comment-538459 + Like, can anything be worse than reading and parsing the text from + /proc/cpuinfo? */ + return Implementation::runtimeFeatures(getauxval(AT_HWCAP)); + + /* Use sysctlbyname() on ARM on macOS / iOS */ + #elif defined(CORRADE_TARGET_ARM) && defined(CORRADE_TARGET_APPLE) + unsigned int out = 0; + /* https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics, + especially "funny" is how most of the values are getting rid of the NEON + naming, probably because they want to push their proprietary AMX. + Sigh. */ + #ifdef CORRADE_TARGET_32BIT + /* Apple says I should use hw.optional.AdvSIMD instead tho */ + if(appleSysctlByName("hw.optional.neon")) out |= TypeTraits::Index; + /* On 32bit I have no idea how to query FMA / vfpv4 support, so that'll + only be implied if FP16 is available as well. Since I don't think there are many 32bit iOS devices left, that's not worth bothering with. */ + #else + /* To avoid string operations, on 64bit I just assume NEON and FMA being + present, like in the Linux case. Again, for extra security make use of + the CORRADE_TARGET_ defines (which should be always there on ARM64) */ + out |= + #ifdef CORRADE_TARGET_NEON + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_NEON_FMA + TypeTraits::Index| + #endif + 0; + #endif + /* Apple says I should use hw.optional.arm.FEAT_FP16 instead tho */ + if(appleSysctlByName("hw.optional.neon_fp16")) { + /* As noted above, if FP16 is available on 32bit, bite the bullet and + assume FMA is there as well */ + #ifdef CORRADE_TARGET_32BIT + out |= TypeTraits::Index; + #endif + out |= TypeTraits::Index; + } + + return Features{out}; + + /* No other (deinlined) implementation at the moment. The function should + not be even defined here in that case -- it's inlined in the header + instead, including the x86 implementation. */ + #else + #error + #endif +} +#endif + +Utility::Debug& operator<<(Utility::Debug& debug, Features value) { + const bool packed = debug.immediateFlags() >= Utility::Debug::Flag::Packed; + + const Containers::StringView prefix = "|Cpu::"_s.exceptSuffix(packed ? 5 : 0); + + /* First one without the | */ + debug << prefix.exceptPrefix(1) << Utility::Debug::nospace; + if(!value) return debug << "Scalar"_s; + + bool written = false; + #define _c(tag) \ + if(value & tag) { \ + if(!written) written = true; \ + else debug << Utility::Debug::nospace << prefix << Utility::Debug::nospace; \ + debug << TypeTraits::name(); \ + value &= ~tag; \ + } + #ifdef CORRADE_TARGET_X86 + _c(Sse2) + _c(Sse3) + _c(Ssse3) + _c(Sse41) + _c(Sse42) + _c(Avx) + _c(Avx2) + _c(Avx512f) + /* Print the extras at the end so the base instruction set is always first + even in case of Cpu::Default, where it's just one */ + _c(Popcnt) + _c(Lzcnt) + _c(Bmi1) + _c(AvxF16c) + _c(AvxFma) + #elif defined(CORRADE_TARGET_ARM) + _c(Neon) + _c(NeonFma) + _c(NeonFp16) + #elif defined(CORRADE_TARGET_EMSCRIPTEN) + _c(Simd128) + #endif + #undef _c + + if(value) { + if(written) debug << Utility::Debug::nospace << prefix << Utility::Debug::nospace; + debug << (packed ? "" : "Features(") << Utility::Debug::nospace << reinterpret_cast(static_cast(value)) << Utility::Debug::nospace << (packed ? "" : ")"); + } + + return debug; +} + +}} diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h new file mode 100644 index 000000000..dfd86ac53 --- /dev/null +++ b/src/Corrade/Cpu.h @@ -0,0 +1,3187 @@ +#ifndef Corrade_Cpu_h +#define Corrade_Cpu_h +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +/** @file + * @brief Namespace @ref Corrade::Cpu and related macros + * @m_since_latest + */ + +#include "Corrade/Corrade.h" +#include "Corrade/Utility/Macros.h" +#include "Corrade/Utility/Utility.h" +#include "Corrade/Utility/visibility.h" + +/* Because can't use inline assembly when targeting 64bit on MSVC, and because + and is just too damn heavy to be included in a + header. Declarations copied verbatim. Clang-cl doesn't like this (undefined + reference to __cpuidex), so using the GCC/Clang codepath on it instead. */ +#if defined(CORRADE_TARGET_MSVC) && !defined(CORRADE_TARGET_CLANG_CL) && defined(CORRADE_TARGET_X86) +extern "C" { + void __cpuidex(int[4], int, int); + unsigned __int64 __cdecl _xgetbv(unsigned int); +} +#endif + +namespace Corrade { + +/** +@brief Compile-time and runtime CPU instruction set detection and dispatch +@m_since_latest + +This namespace provides *tags* for x86, ARM and WebAssembly instruction sets, +which can be used for either system introspection or for choosing a particular +implementation based on the available instruction set. These tags build on top +of the @ref CORRADE_TARGET_SSE2, @ref CORRADE_TARGET_SSE3 etc. preprocessor +macros and provide a runtime feature detection as well. + +This library is built if `WITH_UTILITY` is enabled when building Corrade. To +use this library with CMake, request the `Utility` component of the `Corrade` +package and link to the `Corrade::Utility` target: + +@code{.cmake} +find_package(Corrade REQUIRED Utility) + +# ... +target_link_libraries(your-app PRIVATE Corrade::Utility) +@endcode + +@section Cpu-usage Usage + +The @ref Cpu namespace contains tags such as @ref Cpu::Avx2, @ref Cpu::Sse2, +@ref Cpu::Neon or @ref Cpu::Simd128. These tags behave similarly to enum values +and their combination result in @ref Cpu::Features, which is similar to the +@ref Containers::EnumSet class --- they support the same bitwise operations, +can be tested for subsets and supersets, and are printable with +@ref Utility::Debug. + +The most advanced base CPU instruction set enabled at compile time is then +exposed through the @ref Cpu::DefaultBase variable, which is an alias to one of +those tags, and it matches the architecture-specific @ref CORRADE_TARGET_SSE2 +etc. macros. Since it's a @cpp constexpr @ce variable, it's usable in a +compile-time context. The most straightforward use is shown in the following +C++17 snippet: + +@snippet Corrade-cpp17.cpp Cpu-usage-compile-time + + + +@m_class{m-note m-info} + +@par + If you're writing multiplatform code targeting multiple architectures, you + still need to partially rely on the preprocessor when using the + architecture-specific tags, as those are defined only on the architecture + they apply to. The above would need to be wrapped in + @cpp #ifdef CORRADE_TARGET_X86 @ce; if you would be checking for + @ref Cpu::Neon instead, then you'd need to wrap it in a + @ref CORRADE_TARGET_ARM check. On the other hand, the per-architecture tags + are available on given architecture always --- so for example + @ref Cpu::Avx512f is present even on a compiler that doesn't even recognize + AVX-512 yet. + +@subsection Cpu-usage-dispatch-compile-time Dispatching on available CPU instruction set at compile time + +The main purpose of these tags, however, is to provide means for a compile-time +overload resolution. In other words, picking the best candidate among a set of +functions implemented with various instruction sets. As an example, let's say +you have three different implementations of a certain algorithm transforming +numeric data. One is using AVX2 instructions, another is a slower variant using +just SSE 4.2 and as a fallback there's one with just regular scalar code. To +distinguish them, the functions have the same name, but use a different *tag +type*: + +@snippet Corrade.cpp Cpu-usage-declare + +Then you can either call a particular implementation directly --- for example +to test it --- or you can pass @ref Cpu::DefaultBase, and it'll pick the best +overload candidate for the set of CPU instruction features enabled at compile +time: + +@snippet Corrade.cpp Cpu-usage-compile-time-call + +- If the user code was compiled with AVX2 or higher enabled, the + @ref Cpu::Avx2 overload will be picked. +- Otherwise, if just AVX, SSE 4.2 or anything else that includes SSE 4.2 was + enabled, the @ref Cpu::Sse42 overload will be picked. +- Otherwise (for example when compiling for generic x86-64 that has just the + SSE2 feature set), the @ref Cpu::Scalar overload will be picked. If you + wouldn't provide this overload, the compilation would fail for such a + target --- which is useful for example to enforce a certain CPU feature set + to be enabled in order to use a certain API. + + + +@m_class{m-block m-warning} + +@par SSE3, SSSE3, SSE4.1/SSE4.2, POPCNT, LZCNT, BMI1, AVX F16C and AVX FMA on MSVC + A special case worth mentioning are SSE3 and newer instructions on Windows. + MSVC only provides a very coarse `/arch:SSE2`, `/arch:AVX` and `/arch:AVX2` + for either @ref Sse2, @ref Avx or @ref Avx2, but nothing in between. That + means it's impossible to rely just on compile-time detection to use the + later SSE features on machines that don't support AVX yet (or the various + AVX additions on machines without AVX2), you have to use runtime dispatch + there, as shown below. + +@subsection Cpu-usage-dispatch-runtime Runtime detection and manual dispatch + +So far that was all compile-time detection, which has use mainly when a binary +can be optimized directly for the machine it will run on. But such approach is +not practical when shipping to a heterogenous set of devices. Instead, the +usual workflow is that the majority of code uses the lowest common +denominator (such as SSE2 on x86), with the most demanding functions having +alternative implementations --- picked at runtime --- that make use of more +advanced instructions for better performance. + +Runtime detection is exposed through @ref Cpu::runtimeFeatures(). It will +detect CPU features on platforms that support it, and fall back to +@ref Cpu::compiledFeatures() on platforms that don't. You can then match the +returned @ref Cpu::Features against particular tags to decide which variant to +use: + +@snippet Corrade.cpp Cpu-usage-runtime-manual-dispatch + +While such approach gives you the most control, manually managing the dispatch +branches is error prone and the argument passthrough may also add nontrivial +overhead. See below for an +@ref Cpu-usage-automatic-runtime-dispatch "efficient automatic runtime dispatch". + +@section Cpu-usage-extra Usage with extra instruction sets + +Besides the base instruction set, which on x86 is @ref Sse2 through +@ref Avx512f, with each tag being a superset of the previous one, there are +* *extra* instruction sets such as @ref Popcnt or @ref AvxFma. Basic +compile-time detection for these is still straightforward, only now using +@ref Default instead of @link DefaultBase @endlink: + +@snippet Corrade-cpp17.cpp Cpu-usage-extra-compile-time + +The process of defining and dispatching to function variants that include extra +instruction sets gets moderately more complex, however. As shown on the diagram +below, those are instruction sets that neither fit into the hierarchy nor are +unambiguously included in a later instruction set. For example, some CPUs are +known to have @ref Avx and just @ref AvxFma, some @ref Avx and just +@ref AvxF16c and there are even CPUs with @ref Avx2 but no @ref AvxFma. + +@dotfile cpu.dot + +While there's no possibility of having a total ordering between all possible +combinations for dispatching, the following approach is chosen: + +1. The base instruction set has the main priority. For example, if both an + @ref Avx2 and a @ref Sse2 variant are viable candidates, the @ref Avx2 + variant gets picked, even if the @ref Sse2 variant uses extra + instruction sets that the @ref Avx2 doesn't. +2. After that, the variant with the most extra instruction sets is chosen. For + example, an @ref Avx + @ref AvxFma variant is chosen over plain @ref Avx. + +On the declaration side, the desired base instruction set gets ORed with as +many extra instruction sets as needed, and then wrapped in a +@ref CORRADE_CPU_DECLARE() macro. For example, a lookup algorithm may have a +@ref Sse41 implementation which however also relies on @ref Popcnt and +@ref Lzcnt, and a fallback @ref Sse2 implementation that uses neither: + +@snippet Corrade.cpp Cpu-usage-extra-declare + +And a concrete overload gets picked at compile-time by passing a desired +combination of CPU tags as well --- or @ref Default for the set of features +enabled at compile time --- this time wrapped in a @ref CORRADE_CPU_SELECT(): + +@snippet Corrade.cpp Cpu-usage-extra-compile-time-call + + + +@m_class{m-block m-success} + +@par Resolving overload ambiguity + Because the best overload is picked based on the count of extra instruction + sets used, it may happen that two different variants get assigned the same + priority, causing an ambiguity. For example, the two variants below would + be abiguous for a CPU with @ref Sse41 and both @ref Popcnt and @ref Lzcnt + present: +@par + @snippet Corrade.cpp Cpu-usage-extra-ambiguity +@par + It's not desirable for this library to arbitrarily decide which instruction + set should be preferred --- only the implementation itself can know that. + Thus, to resolve such potential conflict, provide an overload with both + extra tags and delegate from there: +@par + @snippet Corrade.cpp Cpu-usage-extra-ambiguity-resolve + +@section Cpu-usage-target-attributes Enabling instruction sets for particular functions + +On GCC and Clang, a machine target has to be enabled in order to use a +particular CPU instruction set or its intrinsics. While it's possible to do +that for the whole compilation unit by passing for example `-mavx2` to the +compiler, it would force you to create dedicated files for every architecture +variant you want to support. Instead, it's possible to equip particular +functions with *target attributes* defined by @ref CORRADE_ENABLE_SSE2 and +related macros, which then makes a particular instruction set enabled for given +function. + +In contrast, MSVC doesn't restrict intrinsics usage in any way, so you can +freely call e.g. AVX2 intrinsics even if the whole file is compiled with just +SSE2 enabled. The @ref CORRADE_ENABLE_SSE2 and related macros are thus defined +to be empty on this compiler. + +@m_class{m-note m-warning} + +@par + On the other hand, on MSVC, using just the baseline target on the file + level means the compiler will not be able to use any advanced instructions + apart from what you call explicitly via intrinsics. You can try extracting + all AVX+ variants into a dedicated file with `/arch:AVX` enabled and see + if it makes any performance difference. + +For developer convenience, the @ref CORRADE_ENABLE_SSE2 etc. macros are defined +only on matching architectures, and generally only if the compiler itself has +given feature set implemented and usable. Which means you can easily use them +to @cpp #ifdef @ce your variants to be compiled only where it makes sense, or +even guard intrinsics includes with them to avoid including potentially heavy +headers you won't use anyway. In comparison, using the @ref CORRADE_TARGET_SSE2 +etc. macros would only make the variant available if the whole compilation unit +has a corresponding `-m` or `/arch:` option passed to the compiler. + +Finally, the @ref CORRADE_ENABLE() function allows multiple instruction sets to +be enabled at the same time in a more concise way and consistently on both GCC +and Clang. + +Definitions of the `lookup()` function variants from above would then look like +below with the target attributes added. The extra instruction sets get +explicitly enabled as well, in contrast a scalar variant would have no +target-specific annotations at all. + +@snippet Corrade.cpp Cpu-usage-target-attributes + +@section Cpu-usage-automatic-runtime-dispatch Automatic runtime dispatch + +Similarly to how the best-matching function variant can be picked at compile +time, there's a possibility to do the same at runtime without maintaining a +custom dispatch code for each case +@ref Cpu-usage-dispatch-runtime "as was shown above". To avoid having to +dispatch on every call and to remove the argument passthrough overhead, all +variants need to have the same function signature, separate from the CPU tags. +That's achievable by putting them into lambdas with a common signature, and +returning that lambda from a wrapper function that contains the CPU tag. After +that, a runtime dispatcher function that is created with the +@ref CORRADE_CPU_DISPATCHER_BASE() macro. The @cpp transform() @ce variants +from above would then look like this instead: + +@snippet Corrade.cpp Cpu-usage-automatic-runtime-dispatch-declare + +The macro creates an overload of the same name, but taking @ref Features +instead, and internally dispatches to one of the overloads using the same rules +as in the compile-time dispatch. Which means you can now call it with e.g. +@ref runtimeFeatures(), get a function pointer back and then call it with the +actual arguments: + +@snippet Corrade.cpp Cpu-usage-automatic-runtime-dispatch-call + + + +@m_class{m-block m-danger } + +@par Instruction enabling macros and lambdas + An important difference with the @ref CORRADE_ENABLE_SSE2 "CORRADE_ENABLE_*" + macros is that they now have to go also directly next to the lambda as GCC + [currently doesn't propagate the attributes](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80439) + from the wrapper function to the nested lambda. To make matters worse, + older versions of Clang suffer from the inverse problem and ignore lambda + attributes, so you have to specify them on both the lambda and the wrapper + function. GCC 9.1 to 9.3 also has a [bug where it can't parse attributes on lambdas with a trailing return type](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90333). + The preferrable solution is to not use a trailing return type at the cost + of potentially more verbose @cpp return @ce statements. Alternatively you + can require version 8, 9.4 or 10 instead, but note that 9.3 is the default + compiler on Ubuntu 20.04. +@par + All things considered, the above AVX variant would look like this with + relevant macros added: +@par + @snippet Corrade.cpp Cpu-usage-automatic-runtime-dispatch-target-attributes + +@subsection Cpu-usage-automatic-runtime-dispatch-extra Automatic runtime dispach with extra instruction sets + +If the variants are tagged with extra instruction sets instead of just the +base instruction set like in the @cpp lookup() @ce case +@ref Cpu-usage-extra "shown above", you'll use the @ref CORRADE_CPU_DISPATCHER() +macro instead. There, to avoid a combinatorial explosion of cases to check, +you're expected to list the actual extra tags the overloads use. Which is +usually just one or two out of the whole set: + +@snippet Corrade.cpp Cpu-usage-automatic-runtime-dispatch-extra-declare + +On the call side, there's no difference. The created dispatcher function takes +@ref Features as well. + +@section Cpu-usage-automatic-cached-dispatch Automatic cached dispatch + +Ultimately, the dispatch can be performed implicitly, exposing only the final +function or a function pointer, with no additional steps needed from the user +side. There's three possible scenarios with varying performance tradeoffs. +Continuing from the @cpp lookupImplementation() @ce example above: + +
    +
  • +On Linux and Android with API 30+ it's possible to use the +[GNU IFUNC](https://sourceware.org/glibc/wiki/GNU_IFUNC) mechanism, where the +dynamic linker performs a dispatch during the early startup. This is the +fastest variant of runtime dispatch, as it results in an equivalent of a +regular dynamic library function call. Assuming a dispatcher was created using +either @ref CORRADE_CPU_DISPATCHER() or @ref CORRADE_CPU_DISPATCHER_BASE(), +it's implemented using the @ref CORRADE_CPU_DISPATCHED_IFUNC() macro: + +@snippet Corrade.cpp Cpu-usage-automatic-cached-dispatch-ifunc +
    • +
  • +On platforms where IFUNC isn't available, a function pointer can be used +for runtime dispatch instead. It's one additional indirection, which may have a +visible effect if the dispatched-to code is relatively tiny and is called from +within a tight loop. Assuming a dispatcher was created using +either @ref CORRADE_CPU_DISPATCHER() or @ref CORRADE_CPU_DISPATCHER_BASE(), +it's implemented using the @ref CORRADE_CPU_DISPATCHED_POINTER() macro: + +@snippet Corrade.cpp Cpu-usage-automatic-cached-dispatch-pointer +
    • +
  • +For the least amount of overhead, the compile-time dispatch can be used, with +arguments passed through by hand. Similarly to IFUNC, this will also result in +a regular function, but without the indirect overhead. Furthermore, since it's +a direct call to the lambda inside, compiler optimizations will fully inline +its contents, removing any remaining overhead and allowing LTO and other +inter-procedural optimizations that wouldn't be possible with the indirect +calls. This option is best suited for scenarios where it's possible to build +and optimize code for a single target platform. In this case it calls directly +to the original variants, so no macro is needed and +@ref CORRADE_CPU_DISPATCHER() / @ref CORRADE_CPU_DISPATCHER_BASE() is not +needed either: + +@snippet Corrade.cpp Cpu-usage-automatic-cached-dispatch-compile-time +
    • +
+ +With all three cases, you end up with either a function or a function pointer. +The macro signatures are deliberately similar to each other and to the direct +function declaration to make it possible to unify them under a single wrapper +macro in case a practical use case needs to handle more than one variant. + +Finally, when exposed in a header as appropriate, both the function and the +function pointer variant can be then called the same way: + +@snippet Corrade.cpp Cpu-usage-automatic-cached-dispatch-call +*/ +namespace Cpu { + +/** +@brief Traits class for CPU detection tag types + +Useful for detecting tag properties at compile time without the need for +repeated code such as method overloading, cascaded ifs or template +specializations for all tag types. All tag types in the @ref Cpu namespace +have this class implemented. +@see @ref tag(), @ref features() +*/ +#ifndef DOXYGEN_GENERATING_OUTPUT +template struct TypeTraits; +#else +template struct TypeTraits { + enum: unsigned int { + /** + * Tag-specific index. Implementation-defined, is unique among all tags + * on given platform. + */ + Index + }; + + /** + * @brief Tag name + * + * Returns a string representation of the tag, such as @cpp "Avx2" @ce + * for @ref Avx2. + */ + static const char* name(); +}; +#endif + +namespace Implementation { + /* A common type used in all tag constructors to avoid ambiguous calls when + using {} */ + struct InitT {}; + constexpr InitT Init{}; + + enum: unsigned int { ExtraTagBitOffset = 16 }; +} + +/** +@brief Scalar tag type + +See the @ref Cpu namespace and the @ref Scalar tag for more information. +@see @ref tag(), @ref features() +*/ +struct ScalarT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit ScalarT(Implementation::InitT) {} + #endif +}; + +#ifndef DOXYGEN_GENERATING_OUTPUT +/* Scalar code is when nothing else is available, thus no bits set */ +template<> struct TypeTraits { + enum: unsigned int { Index = 0 }; + static const char* name() { return "Scalar"; } +}; +#endif + +#if defined(CORRADE_TARGET_X86) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief SSE2 tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Sse2 tag for more information. +@see @ref tag(), @ref features() +*/ +struct Sse2T: ScalarT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Sse2T(Implementation::InitT): ScalarT{Implementation::Init} {} + #endif +}; + +/** +@brief SSE3 tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Sse3 tag for more information. +@see @ref tag(), @ref features() +*/ +struct Sse3T: Sse2T { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Sse3T(Implementation::InitT): Sse2T{Implementation::Init} {} + #endif +}; + +/** +@brief SSSE3 tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Ssse3 tag for more information. +@see @ref tag(), @ref features() +*/ +struct Ssse3T: Sse3T { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Ssse3T(Implementation::InitT): Sse3T{Implementation::Init} {} + #endif +}; + +/** +@brief SSE4.1 tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Sse41T tag for more information. +@see @ref tag(), @ref features() +*/ +struct Sse41T: Ssse3T { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Sse41T(Implementation::InitT): Ssse3T{Implementation::Init} {} + #endif +}; + +/** +@brief SSE4.2 tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Sse42T tag for more information. +@see @ref tag(), @ref features() +*/ +struct Sse42T: Sse41T { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Sse42T(Implementation::InitT): Sse41T{Implementation::Init} {} + #endif +}; + +/** +@brief POPCNT tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Popcnt tag for more information. +@see @ref tag(), @ref features() +*/ +struct PopcntT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit PopcntT(Implementation::InitT) {} + #endif +}; + +/** +@brief LZCNT tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Lzcnt tag for more information. +@see @ref tag(), @ref features() +*/ +struct LzcntT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit LzcntT(Implementation::InitT) {} + #endif +}; + +/** +@brief BMI1 tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Bmi1 tag for more information. +@see @ref tag(), @ref features() +*/ +struct Bmi1T { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Bmi1T(Implementation::InitT) {} + #endif +}; + +/** +@brief AVX tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Avx tag for more information. +@see @ref tag(), @ref features() +*/ +struct AvxT: Sse42T { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit AvxT(Implementation::InitT): Sse42T{Implementation::Init} {} + #endif +}; + +/** +@brief AVX F16C tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref AvxF16c tag for more information. +@see @ref tag(), @ref features() +*/ +struct AvxF16cT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit AvxF16cT(Implementation::InitT) {} + #endif +}; + +/** +@brief AVX FMA tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref AvxFma tag for more information. +@see @ref tag(), @ref features() +*/ +struct AvxFmaT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit AvxFmaT(Implementation::InitT) {} + #endif +}; + +/** +@brief AVX2 tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Avx2 tag for more information. +@see @ref tag(), @ref features() +*/ +struct Avx2T: AvxT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Avx2T(Implementation::InitT): AvxT{Implementation::Init} {} + #endif +}; + +/** +@brief AVX-512 Foundation tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Avx512f tag for more information. +@see @ref tag(), @ref features() +*/ +struct Avx512fT: Avx2T { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Avx512fT(Implementation::InitT): Avx2T{Implementation::Init} {} + #endif +}; + +#ifndef DOXYGEN_GENERATING_OUTPUT +/* Features earlier in the hierarchy should have lower bits set */ +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 0 }; + static const char* name() { return "Sse2"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 1 }; + static const char* name() { return "Sse3"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 2 }; + static const char* name() { return "Ssse3"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 3 }; + static const char* name() { return "Sse41"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 4 }; + static const char* name() { return "Sse42"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 5 }; + static const char* name() { return "Avx"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 6 }; + static const char* name() { return "Avx2"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 7 }; + static const char* name() { return "Avx512f"; } +}; + +/* The total bit range should not be larger than ExtraTagCount */ +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << (0 + Implementation::ExtraTagBitOffset) }; + static const char* name() { return "Popcnt"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << (1 + Implementation::ExtraTagBitOffset) }; + static const char* name() { return "Lzcnt"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << (2 + Implementation::ExtraTagBitOffset) }; + static const char* name() { return "Bmi1"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << (3 + Implementation::ExtraTagBitOffset) }; + static const char* name() { return "AvxF16c"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << (4 + Implementation::ExtraTagBitOffset) }; + static const char* name() { return "AvxFma"; } +}; +#endif +#endif + +#if defined(CORRADE_TARGET_ARM) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief NEON tag type + +Available only on @ref CORRADE_TARGET_ARM "ARM". See the @ref Cpu namespace +and the @ref Neon tag for more information. +@see @ref tag(), @ref features() +*/ +struct NeonT: ScalarT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit NeonT(Implementation::InitT): ScalarT{Implementation::Init} {} + #endif +}; + +/** +@brief NEON FMA tag type + +Available only on @ref CORRADE_TARGET_ARM "ARM". See the @ref Cpu namespace +and the @ref NeonFma tag for more information. +@see @ref tag(), @ref features() +*/ +struct NeonFmaT: NeonT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit NeonFmaT(Implementation::InitT): NeonT{Implementation::Init} {} + #endif +}; + +/** +@brief NEON FP16 tag type + +Available only on @ref CORRADE_TARGET_ARM "ARM". See the @ref Cpu namespace +and the @ref NeonFp16 tag for more information. +@see @ref tag(), @ref features() +*/ +struct NeonFp16T: NeonFmaT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit NeonFp16T(Implementation::InitT): NeonFmaT{Implementation::Init} {} + #endif +}; + +#ifndef DOXYGEN_GENERATING_OUTPUT +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 0 }; + static const char* name() { return "Neon"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 1 }; + static const char* name() { return "NeonFma"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 2 }; + static const char* name() { return "NeonFp16"; } +}; +#endif +#endif + +#if defined(CORRADE_TARGET_WASM) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief SIMD128 tag type + +Available only on @ref CORRADE_TARGET_WASM "WebAssembly". See the @ref Cpu +namespace and the @ref Simd128 tag for more information. +@see @ref tag(), @ref features() +*/ +struct Simd128T: ScalarT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Simd128T(Implementation::InitT): ScalarT{Implementation::Init} {} + #endif +}; + +#ifndef DOXYGEN_GENERATING_OUTPUT +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 0 }; + static const char* name() { return "Simd128"; } +}; +#endif +#endif + +/** +@brief Scalar tag + +Code that isn't explicitly optimized with any advanced CPU instruction set. +Fallback if no other CPU instruction set is chosen or available. The next most +widely supported instruction sets are @ref Sse2 on x86, @ref Neon on ARM and +@ref Simd128 on WebAssembly. +*/ +constexpr ScalarT Scalar{Implementation::Init}; + +#if defined(CORRADE_TARGET_X86) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief SSE2 tag + +[Streaming SIMD Extensions 2](https://en.wikipedia.org/wiki/SSE2). Available +only on @ref CORRADE_TARGET_X86 "x86", supported by all 64-bit x86 processors +and is present on majority of contemporary 32-bit x86 processors as well. +Superset of @ref Scalar, implied by @ref Sse3. +@see @ref CORRADE_TARGET_SSE2, @ref CORRADE_ENABLE_SSE2 +*/ +constexpr Sse2T Sse2{Implementation::Init}; + +/** +@brief SSE3 tag + +[Streaming SIMD Extensions 3](https://en.wikipedia.org/wiki/SSE3). Available +only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Sse2, implied by +@ref Ssse3. +@see @ref CORRADE_TARGET_SSE3, @ref CORRADE_ENABLE_SSE3 +*/ +constexpr Sse3T Sse3{Implementation::Init}; + +/** +@brief SSSE3 tag + +[Supplemental Streaming SIMD Extensions 3](https://en.wikipedia.org/wiki/SSSE3). +Available only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Sse3, implied +by @ref Sse41. + +Note that certain older AMD processors have [SSE4a](https://en.wikipedia.org/wiki/SSE4#SSE4a) +but neither SSSE3 nor SSE4.1. Both can be however treated as a subset of SSE4.1 +to a large extent, and it's recommended to use @ref Sse41 to handle those. +@see @ref CORRADE_TARGET_SSSE3, @ref CORRADE_ENABLE_SSSE3 +*/ +constexpr Ssse3T Ssse3{Implementation::Init}; + +/** +@brief SSE4.1 tag + +[Streaming SIMD Extensions 4.1](https://en.wikipedia.org/wiki/SSE4#SSE4.1). +Available only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Ssse3, +implied by @ref Sse42. + +Note that certain older AMD processors have [SSE4a](https://en.wikipedia.org/wiki/SSE4#SSE4a) +but neither SSSE3 nor SSE4.1. Both can be however treated as a subset of SSE4.1 +to a large extent, and it's recommended to use @ref Sse41 to handle those. +@see @ref CORRADE_TARGET_SSE41, @ref CORRADE_ENABLE_SSE41 +*/ +constexpr Sse41T Sse41{Implementation::Init}; + +/** +@brief SSE4.2 tag + +[Streaming SIMD Extensions 4.2](https://en.wikipedia.org/wiki/SSE4#SSE4.2). +Available only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Sse41, +implied by @ref Avx. +@see @ref CORRADE_TARGET_SSE42, @ref CORRADE_ENABLE_SSE42 +*/ +constexpr Sse42T Sse42{Implementation::Init}; + +/** +@brief POPCNT tag + +[POPCNT](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation)) +instructions. Available only on @ref CORRADE_TARGET_X86 "x86". This instruction +set is treated as an *extra*, i.e. is neither a superset of nor implied by any +other instruction set. See @ref Cpu-usage-extra for more information. +@see @ref Lzcnt, @ref Bmi1, @ref CORRADE_TARGET_POPCNT, + @ref CORRADE_ENABLE_POPCNT +*/ +constexpr PopcntT Popcnt{Implementation::Init}; + +/** +@brief LZCNT tag + +[LZCNT](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation)) +instructions. Available only on @ref CORRADE_TARGET_X86 "x86". This instruction +set is treated as an *extra*, i.e. is neither a superset of nor implied by any +other instruction set. See @ref Cpu-usage-extra for more information. + +Note that this instruction has encoding compatible with an earlier `BSR` +instruction which has a slightly different behavior. To avoid wrong results if +it isn't available, prefer to always detect its presence with +@ref runtimeFeatures() instead of a compile-time check. +@see @ref Popcnt, @ref Bmi1, @ref CORRADE_TARGET_LZCNT, + @ref CORRADE_ENABLE_LZCNT +*/ +constexpr LzcntT Lzcnt{Implementation::Init}; + +/** +@brief BMI1 tag + +[BMI1](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#BMI1_(Bit_Manipulation_Instruction_Set_1)) +instructions, including `TZCNT`. Available only on +@ref CORRADE_TARGET_X86 "x86". This instruction set is treated as an *extra*, +i.e. is neither a superset of nor implied by any other instruction set. See +@ref Cpu-usage-extra for more information. + +Note that the `TZCNT` instruction has encoding compatible with an earlier `BSF` +instruction which has a slightly different behavior. To avoid wrong results if +it isn't available, prefer to always detect its presence with +@ref runtimeFeatures() instead of a compile-time check. +@see @ref Popcnt, @ref Lzcnt, @ref CORRADE_TARGET_BMI1, + @ref CORRADE_ENABLE_BMI1 +*/ +constexpr Bmi1T Bmi1{Implementation::Init}; + +/** +@brief AVX tag + +[Advanced Vector Extensions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions). +Available only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Sse42, +implied by @ref Avx2. +@see @ref CORRADE_TARGET_AVX, @ref CORRADE_ENABLE_AVX +*/ +constexpr AvxT Avx{Implementation::Init}; + +/** +@brief AVX F16C tag + +[F16C](https://en.wikipedia.org/wiki/F16C) instructions. Available only on +@ref CORRADE_TARGET_X86 "x86". This instruction set is treated as an *extra*, +i.e. is neither a superset of nor implied by any other instruction set. See +@ref Cpu-usage-extra for more information. +@see @ref CORRADE_TARGET_AVX_F16C, @ref CORRADE_ENABLE_AVX_F16C +*/ +constexpr AvxF16cT AvxF16c{Implementation::Init}; + +/** +@brief AVX FMA tag + +[FMA3 instruction set](https://en.wikipedia.org/wiki/FMA_instruction_set). +Available only on @ref CORRADE_TARGET_X86 "x86". This instruction set is +treated as an *extra*, i.e. is neither a superset of nor implied by any other +instruction set. See @ref Cpu-usage-extra for more information. +@see @ref CORRADE_TARGET_AVX_FMA, @ref CORRADE_ENABLE_AVX_FMA +*/ +constexpr AvxFmaT AvxFma{Implementation::Init}; + +/** +@brief AVX2 tag + +[Advanced Vector Extensions 2](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#Advanced_Vector_Extensions_2). +Available only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Avx, +implied by @ref Avx512f. +@see @ref CORRADE_TARGET_AVX2, @ref CORRADE_ENABLE_AVX2 +*/ +constexpr Avx2T Avx2{Implementation::Init}; + +/** +@brief AVX-512 Foundation tag + +[AVX-512](https://en.wikipedia.org/wiki/AVX-512) Foundation. Available only on +@ref CORRADE_TARGET_X86 "x86". Superset of @ref Avx2. +@see @ref CORRADE_TARGET_AVX512F, @ref CORRADE_ENABLE_AVX512F +*/ +constexpr Avx512fT Avx512f{Implementation::Init}; +#endif + +#if defined(CORRADE_TARGET_ARM) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief NEON tag type + +[ARM NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)). +Available only on @ref CORRADE_TARGET_ARM "ARM". Superset of @ref Scalar, +implied by @ref NeonFp16. +@see @ref CORRADE_TARGET_NEON, @ref CORRADE_ENABLE_NEON +*/ +constexpr NeonT Neon{Implementation::Init}; + +/** +@brief NEON FMA tag type + +[ARM NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)) +with FMA instructions. Available only on @ref CORRADE_TARGET_ARM "ARM". +Superset of @ref Neon, implied by @ref NeonFp16. +@see @ref CORRADE_TARGET_NEON_FMA, @ref CORRADE_ENABLE_NEON_FMA +*/ +constexpr NeonFmaT NeonFma{Implementation::Init}; + +/** +@brief NEON FP16 tag type + +[ARM NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)) +with ARMv8.2-a FP16 vector arithmetic. Available only on +@ref CORRADE_TARGET_ARM "ARM". Superset of @ref NeonFma. +@see @ref CORRADE_TARGET_NEON_FP16, @ref CORRADE_ENABLE_NEON_FP16 +*/ +constexpr NeonFp16T NeonFp16{Implementation::Init}; +#endif + +#if defined(CORRADE_TARGET_WASM) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief SIMD128 tag type + +[128-bit WebAssembly SIMD](https://github.com/webassembly/simd). Available only +on @ref CORRADE_TARGET_WASM "WebAssembly". Superset of @ref Scalar. +@see @ref CORRADE_TARGET_SIMD128, @ref CORRADE_ENABLE_SIMD128 +*/ +constexpr Simd128T Simd128{Implementation::Init}; +#endif + +namespace Implementation { + +#ifndef DOXYGEN_GENERATING_OUTPUT +/* "warning: Detected potential recursive class relation between..." I DON'T + CARE, DOXYGEN, IT'S NOT YOUR JOB TO PARSE THE CODE IN THIS NAMESPACE */ +template struct Priority: Priority {}; +template<> struct Priority<0> {}; +#endif + +/* Count of "extra" tags that are not in the hierarchy. Should not be larger + than strictly necessary as it deepens inheritance hierarchy when picking + best overload candidate. */ +enum: unsigned int { + BaseTagMask = (1 << ExtraTagBitOffset) - 1, + ExtraTagMask = 0xffffffffu & ~BaseTagMask, + #ifdef CORRADE_TARGET_X86 + ExtraTagCount = 5, + #else + ExtraTagCount = 0, + #endif +}; + +/* On sane compilers (and on MSVC with /permissive-), these two could be + directly in the Tag constructor enable_if expressions. But MSVC chokes hard + on those ("C2988: unrecognizable template declaration/definition", haha), so + they have to be extracted outside. I could also #ifdef around the workaround + and extract it only for MSVC, but I don't think the code duplication would + be worth it. */ +template struct IsTagConversionAllowed { + enum: bool { Value = + /* There should be at most one base tag set in both, if there's none + then it's Cpu::Scalar */ + !((value & BaseTagMask) & ((value & BaseTagMask) - 1)) && + !((otherValue & BaseTagMask) & ((otherValue & BaseTagMask) - 1)) && + /* The other base tag should be the same or derived (i.e, having same + or larger value) */ + (otherValue & BaseTagMask) >= (value & BaseTagMask) && + /* The other extra bits should be a superset of this */ + ((otherValue & value) & ExtraTagMask) == (value & ExtraTagMask) + }; +}; +template struct IsSingleTagConversionAllowed { + enum: bool { Value = + /* There should be at most one base tag set in this one, the other + satisfies that implicitly as it's constrained by TypeTraits */ + !((value & BaseTagMask) & ((value & BaseTagMask) - 1)) && + /* The other base tag should be the same or derived (i.e, having same + or larger value) */ + (otherIndex & BaseTagMask) >= (value & BaseTagMask) && + /* The other extra bits should be a superset of this. Since a single + tag can be only one bit, this condition gets satisfied only either + if we're Cpu::Scalar or if we have no extra bits. */ + ((otherIndex & value) & ExtraTagMask) == (value & ExtraTagMask) + }; +}; + +/* Holds a compile-time combination of tags. Kept private, since it isn't + really directly needed in user code and it would only lead to confusion. */ +template struct Tags { + enum: unsigned int { Value = value }; + + /* Empty initialization. Should not be needed by public code. */ + constexpr explicit Tags(InitT) {} + + /* Conversion from other tag combination, allowed only if the other is + not a subset */ + template constexpr Tags(Tags, typename std::enable_if::Value>::type* = {}) {} + + /* Conversion from a single tag, allowed only if we're a single bit and + the other is not a subset */ + template constexpr Tags(T, typename std::enable_if::Index>::Value>::type* = {}) {} + + /* A subset of operators on Features, excluding the assignment ones -- + since they modify the type, they make no sense here */ + template constexpr Tags operator|(Tags) const { + return Tags{Init}; + } + template constexpr Tags::Index> operator|(U) const { + return Tags::Index>{Init}; + } + template constexpr Tags operator&(Tags) const { + return Tags{Init}; + } + template constexpr Tags::Index> operator&(U) const { + return Tags::Index>{Init}; + } + template constexpr Tags operator^(Tags) const { + return Tags{Init}; + } + template constexpr Tags::Index> operator^(U) const { + return Tags::Index>{Init}; + } + constexpr Tags<~value> operator~() const { + return Tags<~value>{Init}; + } + constexpr explicit operator bool() const { return value; } + constexpr operator unsigned int() const { return value; } +}; + +template constexpr Tags::Index> tags(T) { + return Tags::Index>{Init}; +} +template constexpr Tags tags(Tags tags) { + return tags; +} + +/* Base-2 log, "plus one" (returns 0 for A == 0). For a base tag, which is + always just one bit, returns position of that bit. Used for calculating + distance between two tags in order to calculate overload priority. But since + there's also the Scalar tag, which is 0, it's plus one. */ +template struct BitIndex { + enum: unsigned short { Value = 1 + BitIndex<(A >> 1)>::Value }; +}; +template<> struct BitIndex<0> { + enum: unsigned short { Value = 0 }; +}; + +/* Popcount, but constexpr. No, not related to Cpu::Popcnt, in any way. Used + for calculating a difference between two extra tag sets in order to + calculate overload priority. */ +template struct BitCount { + /* https://en.wikipedia.org/wiki/Hamming_weight#Efficient_implementation; + lol I can do that without having to recurse */ + enum: unsigned short { + Bits1 = 0x5555, /* 0b0101010101010101 */ + Bits2 = 0x3333, /* 0b0011001100110011 */ + Bits4 = 0x0f0f, /* 0b0000111100001111 */ + Bits8 = 0x00ff, /* 0b0000000011111111 */ + + B0 = (A >> 0) & Bits1, + B1 = (A >> 1) & Bits1, + C = B0 + B1, + D0 = (C >> 0) & Bits2, + D2 = (C >> 2) & Bits2, + E = D0 + D2, + F0 = (E >> 0) & Bits4, + F4 = (E >> 4) & Bits4, + G = F0 + F4, + H0 = (G >> 0) & Bits8, + H8 = (G >> 8) & Bits8, + Value = H0 + H8 + }; +}; + +/* Calculates an absolute priority index for given tag, which is either a + base-2 log "plus one" of its index if it's a base tag, or is 1 if it's an + extra tag. + + MSVC 2015 and 2017 need the extra () inside Priority<>, otherwise they + demand that a typename is used for TypeTraits::Index. Heh. */ +template Priority<(TypeTraits::Index & ExtraTagMask ? 1 : BitIndex::Index & BaseTagMask>::Value*(ExtraTagCount + 1))> constexpr priority(T) { + return {}; +} +template Priority<(BitIndex::Value*(ExtraTagCount + 1) + BitCount<((value & ExtraTagMask) >> ExtraTagBitOffset)>::Value)> constexpr priority(Tags) { + static_assert(!((value & BaseTagMask) & ((value & BaseTagMask) - 1)), "more than one base tag used"); + /* GCC 4.8 loudly complains about enum comparison if I don't cast, sigh */ + static_assert(((value & ExtraTagMask) >> ExtraTagBitOffset) < (1 << static_cast(ExtraTagCount)), "extra tag out of expected bounds"); + return {}; +} + +} + +/** +@brief Default base tag type + +See the @ref DefaultBase tag for more information. +*/ +typedef + #ifdef CORRADE_TARGET_X86 + #ifdef CORRADE_TARGET_AVX512F + Avx512fT + #elif defined(CORRADE_TARGET_AVX2) + Avx2T + #elif defined(CORRADE_TARGET_AVX) + AvxT + #elif defined(CORRADE_TARGET_SSE42) + Sse42T + #elif defined(CORRADE_TARGET_SSE41) + Sse41T + #elif defined(CORRADE_TARGET_SSSE3) + Ssse3T + #elif defined(CORRADE_TARGET_SSE3) + Sse3T + #elif defined(CORRADE_TARGET_SSE2) + Sse2T + #else + ScalarT + #endif + + #elif defined(CORRADE_TARGET_ARM) + #ifdef CORRADE_TARGET_NEON_FP16 + NeonFp16T + #elif defined(CORRADE_TARGET_NEON_FMA) + NeonFmaT + #elif defined(CORRADE_TARGET_NEON) + NeonT + #else + ScalarT + #endif + + #elif defined(CORRADE_TARGET_WASM) + #ifdef CORRADE_TARGET_SIMD128 + Simd128T + #else + ScalarT + #endif + #endif + DefaultBaseT; + +/** +@brief Default extra tag type + +See the @ref DefaultExtra tag for more information. +*/ +typedef Implementation::Tags< + #ifdef CORRADE_TARGET_X86 + #ifdef CORRADE_TARGET_POPCNT + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_LZCNT + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_BMI1 + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_AVX_FMA + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_AVX_F16C + TypeTraits::Index| + #endif + #endif + 0> DefaultExtraT; + +/** +@brief Default tag type + +See the @ref Default tag for more information. +*/ +typedef Implementation::Tags::Index|DefaultExtraT::Value> DefaultT; + +/** +@brief Default base tag + +Highest base instruction set available on given architecture with current +compiler flags. Ordered by priority, on @ref CORRADE_TARGET_X86 it's one of these: + +- @ref Avx512f if @ref CORRADE_TARGET_AVX512F is defined +- @ref Avx2 if @ref CORRADE_TARGET_AVX2 is defined +- @ref Avx if @ref CORRADE_TARGET_AVX is defined +- @ref Sse42 if @ref CORRADE_TARGET_SSE42 is defined +- @ref Sse41 if @ref CORRADE_TARGET_SSE41 is defined +- @ref Ssse3 if @ref CORRADE_TARGET_SSSE3 is defined +- @ref Sse3 if @ref CORRADE_TARGET_SSE3 is defined +- @ref Sse2 if @ref CORRADE_TARGET_SSE2 is defined +- @ref Scalar otherwise + +On @ref CORRADE_TARGET_ARM it's one of these: + +- @ref NeonFp16 if @ref CORRADE_TARGET_NEON_FP16 is defined +- @ref NeonFma if @ref CORRADE_TARGET_NEON_FMA is defined +- @ref Neon if @ref CORRADE_TARGET_NEON is defined +- @ref Scalar otherwise + +On @ref CORRADE_TARGET_WASM it's one of these: + +- @ref Simd128 if @ref CORRADE_TARGET_SIMD128 is defined +- @ref Scalar otherwise + +In addition to the above, @ref DefaultExtra contains a combination of extra +instruction sets available together with the base instruction set, and +@ref Default is a combination of both. See also @ref compiledFeatures() which +returns a *combination* of base tags instead of just the highest available, +together with the extra instruction sets, and @ref runtimeFeatures() which is +capable of detecting the available CPU feature set at runtime. +*/ +constexpr DefaultBaseT DefaultBase{Implementation::Init}; + +/** +@brief Default extra tags + +Instruction sets available in addition to @ref DefaultBase on +given architecture with current compiler flags. On @ref CORRADE_TARGET_X86 it's +a combination of these: + +- @ref Popcnt if @ref CORRADE_TARGET_POPCNT is defined +- @ref Lzcnt if @ref CORRADE_TARGET_LZCNT is defined +- @ref Bmi1 if @ref CORRADE_TARGET_BMI1 is defined +- @ref AvxFma if @ref CORRADE_TARGET_AVX_FMA is defined +- @ref AvxF16c if @ref CORRADE_TARGET_AVX_F16C is defined + +No extra instruction sets are currently defined for @ref CORRADE_TARGET_ARM or +@ref CORRADE_TARGET_WASM. + +In addition to the above, @ref Default is a combination of both +@ref DefaultBase and the extra instruction sets. See also +@ref compiledFeatures() which returns these together with a combination of all +base instruction sets available, and @ref runtimeFeatures() which is capable of +detecting the available CPU feature set at runtime. +*/ +constexpr DefaultExtraT DefaultExtra{Implementation::Init}; + +/** +@brief Default tags + +A combination of @ref DefaultBase and @ref DefaultExtra, see their +documentation for more information. +*/ +constexpr DefaultT Default{Implementation::Init}; + +/** +@brief Tag for a tag type + +Returns a tag corresponding to tag type @p T. The following two expressions are +equivalent: + +@snippet Corrade.cpp Cpu-tag-from-type + +@see @ref features() +*/ +template constexpr T tag() { return T{Implementation::Init}; } + +#if defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) +namespace Implementation { + /* Needed for a friend declaration, implementation is at the very end of + the header */ + Features runtimeFeatures(unsigned long caps); +} +#endif + +/** +@brief Feature set + +Provides storage and comparison as well as runtime detection of CPU instruction +set. Provides an interface similar to an @ref Containers::EnumSet, with values +being the @ref Sse2, @ref Sse3 etc. tags. + +See the @ref Cpu namespace for an overview and usage examples. +@see @ref compiledFeatures(), @ref runtimeFeatures() +*/ +class Features { + public: + /** + * @brief Default constructor + * + * Equivalent to @ref Scalar. + */ + constexpr explicit Features() noexcept: _data{} {} + + /** + * @brief Construct from a tag + * + * @see @ref features() + */ + template::Index)> constexpr /*implicit*/ Features(T) noexcept: _data{TypeTraits::Index} { + /* GCC 4.8 loudly complains about enum comparison if I don't cast, sigh */ + static_assert(((TypeTraits::Index & Implementation::ExtraTagMask) >> Implementation::ExtraTagBitOffset) < (1 << static_cast(Implementation::ExtraTagCount)), + "extra tag out of expected bounds"); + } + + #ifndef DOXYGEN_GENERATING_OUTPUT + /* The compile-time Tags<> is an implementation detail, don't show that + in the docs */ + template constexpr /*implicit*/ Features(Implementation::Tags) noexcept: _data{value} {} + #endif + + /** @brief Equality comparison */ + constexpr bool operator==(Features other) const { + return _data == other._data; + } + + /** @brief Non-equality comparison */ + constexpr bool operator!=(Features other) const { + return _data != other._data; + } + + /** + * @brief Whether @p other is a subset of this (@f$ a \supseteq o @f$) + * + * Equivalent to @cpp (a & other) == other @ce. + */ + constexpr bool operator>=(Features other) const { + return (_data & other._data) == other._data; + } + + /** + * @brief Whether @p other is a superset of this (@f$ a \subseteq o @f$) + * + * Equivalent to @cpp (a & other) == a @ce. + */ + constexpr bool operator<=(Features other) const { + return (_data & other._data) == _data; + } + + /** @brief Union of two feature sets */ + constexpr Features operator|(Features other) const { + return Features{_data | other._data}; + } + + /** @brief Union two feature sets and assign */ + Features& operator|=(Features other) { + _data |= other._data; + return *this; + } + + /** @brief Intersection of two feature sets */ + constexpr Features operator&(Features other) const { + return Features{_data & other._data}; + } + + /** @brief Intersect two feature sets and assign */ + Features& operator&=(Features other) { + _data &= other._data; + return *this; + } + + /** @brief XOR of two feature sets */ + constexpr Features operator^(Features other) const { + return Features{_data ^ other._data}; + } + + /** @brief XOR two feature sets and assign */ + Features& operator^=(Features other) { + _data ^= other._data; + return *this; + } + + /** @brief Feature set complement */ + constexpr Features operator~() const { + return Features{~_data}; + } + + /** + * @brief Boolean conversion + * + * Returns @cpp true @ce if at least one feature apart from @ref Scalar + * is present, @cpp false @ce otherwise. + */ + constexpr explicit operator bool() const { return _data; } + + /** + * @brief Integer representation + * + * For testing purposes. @ref Cpu::Scalar is always @cpp 0 @ce, values + * corresponding to other feature tags are unspecified. + */ + constexpr explicit operator unsigned int() const { return _data; } + + private: + template friend constexpr Features features(); + friend constexpr Features compiledFeatures(); + #if (defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC))) || (defined(CORRADE_TARGET_ARM) && defined(CORRADE_TARGET_APPLE)) + friend + #ifdef CORRADE_TARGET_ARM + /* MSVC demands the export macro to be here as well. Inlined on x86. */ + CORRADE_UTILITY_EXPORT + #endif + Features runtimeFeatures(); + #endif + #if defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) + friend Features Implementation::runtimeFeatures(unsigned long); + #endif + + constexpr explicit Features(unsigned int data) noexcept: _data{data} {} + + unsigned int _data; +}; + +/** +@brief Feature set for a tag type + +Returns @ref Features with a tag corresponding to tag type @p T, avoiding a +need to form the tag value in order to pass it to @ref Features::Features(T). +The following two expressions are equivalent: + +@snippet Corrade.cpp Cpu-features-from-type + +@see @ref tag() +*/ +template constexpr Features features() { + return Features{TypeTraits::Index}; +} + +/** @relates Features +@brief Equality comparison of a tag and a feature set + +Same as @ref Features::operator==(). +*/ +template::Index)> constexpr bool operator==(T a, Features b) { + return Features(a) == b; +} + +/** @relates Features +@brief Non-equality comparison of a tag and a feature set + +Same as @ref Features::operator!=(). +*/ +template::Index)> constexpr bool operator!=(T a, Features b) { + return Features(a) != b; +} + +/** @relates Features +@brief Whether @p a is a superset of @p b (@f$ a \supseteq b @f$) + +Same as @ref Features::operator>=(). +*/ +template::Index)> constexpr bool operator>=(T a, Features b) { + return Features(a) >= b; +} + +/** @relates Features +@brief Whether @p a is a subset of @p b (@f$ a \subseteq b @f$) + +Same as @ref Features::operator<=(). +*/ +template::Index)> constexpr bool operator<=(T a, Features b) { + return Features(a) <= b; +} + +/** @relates Features +@brief Union of two feature sets + +Same as @ref Features::operator|(). +*/ +template::Index)> constexpr Features operator|(T a, Features b) { + return b | a; +} + +#ifndef DOXYGEN_GENERATING_OUTPUT +/* Compared to the above, this produces a type that encodes the value instead + of Features. Has to be in the same namespace as the tags, but since Tags<> + is an implementation detail, this is hidden from plain sight as well. */ +template constexpr Implementation::Tags::Index | TypeTraits::Index> operator|(T, U) { + return Implementation::Tags::Index | TypeTraits::Index>{Implementation::Init}; +} +template constexpr Implementation::Tags::Index | value> operator|(T, Implementation::Tags) { + return Implementation::Tags::Index | value>{Implementation::Init}; +} +#endif + +/** @relates Features +@brief Intersection of two feature sets + +Same as @ref Features::operator&(). +*/ +template::Index)> constexpr Features operator&(T a, Features b) { + return b & a; +} + +#ifndef DOXYGEN_GENERATING_OUTPUT +/* Compared to the above, this produces a type that encodes the value instead + of Features. Has to be in the same namespace as the tags, but since Tags<> + is an implementation detail, this is hidden from plain sight as well. */ +template constexpr Implementation::Tags::Index & TypeTraits::Index> operator&(T, U) { + return Implementation::Tags::Index & TypeTraits::Index>{Implementation::Init}; +} +template constexpr Implementation::Tags::Index & value> operator&(T, Implementation::Tags) { + return Implementation::Tags::Index & value>{Implementation::Init}; +} +#endif + +/** @relates Features +@brief XOR of two feature sets + +Same as @ref Features::operator^(). +*/ +template::Index)> constexpr Features operator^(T a, Features b) { + return b ^ a; +} + +#ifndef DOXYGEN_GENERATING_OUTPUT +/* Compared to the above, this produces a type that encodes the value instead + of Features. Has to be in the same namespace as the tags, but since Tags<> + is an implementation detail, this is hidden from plain sight as well. */ +template constexpr Implementation::Tags::Index ^ TypeTraits::Index> operator^(T, U) { + return Implementation::Tags::Index ^ TypeTraits::Index>{Implementation::Init}; +} +template constexpr Implementation::Tags::Index ^ value> operator^(T, Implementation::Tags) { + return Implementation::Tags::Index ^ value>{Implementation::Init}; +} +#endif + +/** @relates Features +@brief Feature set complement + +Same as @ref Features::operator~(). +*/ +#ifdef DOXYGEN_GENERATING_OUTPUT +template constexpr Features operator~(T a); +#else +/* To avoid confusion, to the doc use it's shown that the operator produces a + Features, but in fact it's a type with a compile-time-encoded value */ +template constexpr Implementation::Tags<~TypeTraits::Index> operator~(T) { + return Implementation::Tags<~TypeTraits::Index>{Implementation::Init}; +} +#endif + +/** @debugoperator{Features} */ +CORRADE_UTILITY_EXPORT Utility::Debug& operator<<(Utility::Debug& debug, Features value); + +/** @relates Features +@overload +*/ +template::Index)> inline Utility::Debug& operator<<(Utility::Debug& debug, T value) { + return operator<<(debug, Features{value}); +} + +namespace Implementation { + template inline Utility::Debug& operator<<(Utility::Debug& debug, Tags value) { + return operator<<(debug, Features{value}); + } +} + +/** +@brief CPU instruction sets enabled at compile time + +On @ref CORRADE_TARGET_X86 "x86" returns a combination of @ref Sse2, @ref Sse3, +@ref Ssse3, @ref Sse41, @ref Sse42, @ref Popcnt, @ref Lzcnt, @ref Bmi1, +@ref Avx, @ref AvxF16c, @ref AvxFma, @ref Avx2 and @ref Avx512f based on what +all @ref CORRADE_TARGET_SSE2 etc. preprocessor variables are defined. + +On @ref CORRADE_TARGET_ARM "ARM", returns a combination of @ref Neon, +@ref NeonFma and @ref NeonFp16 based on what all @ref CORRADE_TARGET_NEON etc. +preprocessor variables are defined. + +On @ref CORRADE_TARGET_WASM "WebAssembly", returns @ref Simd128 based on +whether the @ref CORRADE_TARGET_SIMD128 preprocessor variable is defined. + +On other platforms or if no known CPU instruction set is enabled, the returned +value is equal to @ref Scalar, which in turn is equivalent to empty (or +default-constructed) @ref Features. +@see @ref DefaultBase, @ref DefaultExtra, @ref Default +*/ +constexpr Features compiledFeatures() { + return Features{ + #ifdef CORRADE_TARGET_X86 + #ifdef CORRADE_TARGET_SSE2 + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_SSE3 + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_SSSE3 + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_SSE41 + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_SSE42 + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_POPCNT + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_LZCNT + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_BMI1 + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_AVX + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_AVX_FMA + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_AVX_F16C + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_AVX2 + TypeTraits::Index| + #endif + + #elif defined(CORRADE_TARGET_ARM) + #ifdef CORRADE_TARGET_NEON + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_NEON_FMA + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_NEON_FP16 + TypeTraits::Index| + #endif + + #elif defined(CORRADE_TARGET_WASM) + #ifdef CORRADE_TARGET_SIMD128 + TypeTraits::Index| + #endif + #endif + 0}; +} + +/** +@brief Detect available CPU instruction sets at runtime + +On @ref CORRADE_TARGET_X86 "x86" and GCC, Clang or MSVC uses the +[CPUID](https://en.wikipedia.org/wiki/CPUID) builtin to check for the +@ref Sse2, @ref Sse3, @ref Ssse3, @ref Sse41, @ref Sse42, @ref Popcnt, +@ref Lzcnt, @ref Bmi1, @ref Avx, @ref AvxF16c, @ref AvxFma, @ref Avx2 and +@ref Avx512f runtime features. @ref Avx needs OS support as well, if it's not +present, no following flags including @ref Bmi1 are checked either. On +compilers other than GCC, Clang and MSVC the function is @cpp constexpr @ce and +delegates into @ref compiledFeatures(). + +On @ref CORRADE_TARGET_ARM "ARM" and Linux or Android API level 18+ uses +@m_class{m-doc-external} [getauxval()](https://man.archlinux.org/man/getauxval.3), or on ARM macOS and iOS uses @m_class{m-doc-external} [sysctlbyname()](https://developer.apple.com/documentation/kernel/1387446-sysctlbyname) +to check for the @ref Neon, @ref NeonFma and @ref NeonFp16. @ref Neon and +@ref NeonFma are implicitly supported on ARM64. On other platforms the function +is @cpp constexpr @ce and delegates into @ref compiledFeatures(). + +On @ref CORRADE_TARGET_WASM "WebAssembly" an attempt to use SIMD instructions +without runtime support results in a WebAssembly compilation error and thus +runtime detection is largely meaningless. While this may change once the +[feature detection proposal](https://github.com/WebAssembly/feature-detection/blob/main/proposals/feature-detection/Overview.md) +is implemented, at the moment the function is @cpp constexpr @ce and delegates +into @ref compiledFeatures(). + +On other platforms or if no known CPU instruction set is detected, the returned +value is equal to @ref Scalar, which in turn is equivalent to empty (or +default-constructed) @ref Features. +@see @ref DefaultBase, @ref DefaultExtra, @ref Default +*/ +#if (defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC))) || (defined(CORRADE_TARGET_ARM) && ((defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18)) || defined(CORRADE_TARGET_APPLE))) || defined(DOXYGEN_GENERATING_OUTPUT) +#ifdef CORRADE_TARGET_ARM +CORRADE_UTILITY_EXPORT /* Inlined on x86 at the very end of the header */ +#endif +Features runtimeFeatures(); +#else +constexpr Features runtimeFeatures() { return compiledFeatures(); } +#endif + +/** +@brief Declare a CPU tag for a compile-time dispatch +@m_since_latest + +Meant to be used to declare a function overload that uses given combination of +CPU instruction sets. The @ref CORRADE_CPU_SELECT() macro is a counterpart used +to select among overloads declared with this macro. See @ref Cpu-usage-extra +for more information and usage example. + +Internally, this macro expands to two function parameter declarations separated +by a comma, one that ensures only an overload matching the desired instruction +sets get picked, and one that assigns an absolute priority to this overload. +*/ +#define CORRADE_CPU_DECLARE(tag) decltype(Corrade::Cpu::Implementation::tags(tag)), decltype(Corrade::Cpu::Implementation::priority(tag)) + +/** +@brief Select a CPU tag for a compile-time dispatch +@m_since_latest + +Meant to be used to select among function overloads declared with +@ref CORRADE_CPU_DECLARE() that best matches given combination of CPU +instruction sets. See @ref Cpu-usage-extra for more information and usage +example. + +Internally, this macro expands to two function parameter values separated by a +comma, one that contains the desired instruction sets to filter the overloads +against and another that converts the sets to an absolute priority to pick the +best viable overload. +*/ +#define CORRADE_CPU_SELECT(tag) tag, Corrade::Cpu::Implementation::priority(tag) + +#ifndef DOXYGEN_GENERATING_OUTPUT +/* Called from CORRADE_CPU_DISPATCHER() and _CORRADE_ENABLE_CONCATENATE() to + pick a macro implementation based on how many arguments were passed. Source: + https://stackoverflow.com/a/11763277 */ +/** @todo move to Utility/Macros.h once it gets useful elsewhere */ +#define _CORRADE_HELPER_PICK(_0, _1, _2, _3, _4, _5, _6, _7, macroName, ...) macroName +#endif + +/** +@brief Create a function for a runtime dispatch on a base CPU instruction set +@m_since_latest + +Given a set of function overloads named @p function that accept a CPU tag as a +parameter, all returning a function pointer of the same type, creates a +function with signature @cpp function(Cpu::Features) @ce which will select +among the overloads using a runtime-specified +@relativeref{Corrade,Cpu::Features}, using the same rules as the compile-time +overload selection. For this macro to work, at the very least there has to be +an overload with a @relativeref{Corrade,Cpu::ScalarT} argument. See +@ref Cpu-usage-automatic-runtime-dispatch for more information and an example. + +This function works with just a single base CPU instruction tag such as +@relativeref{Corrade,Cpu::Avx2} or @relativeref{Corrade,Cpu::Neon}, but not the +extra instruction sets like @relativeref{Corrade,Cpu::Lzcnt} or +@relativeref{Corrade,Cpu::AvxFma}. For a dispatch that takes extra instruction +sets into account as well use @ref CORRADE_CPU_DISPATCHER() instead. +*/ +/* Ideally this would reuse _CORRADE_CPU_DISPATCHER_IMPLEMENTATION(), + unfortunately due to MSVC not being able to defer macro calls unless + "the new preprocessor" is enabled it wouldn't be possible to get rid of the + CORRADE_CPU_SELECT() macro from there. */ +#ifdef CORRADE_TARGET_X86 +#define CORRADE_CPU_DISPATCHER_BASE(function) \ + decltype(function(Corrade::Cpu::Scalar)) function(Corrade::Cpu::Features features) { \ + if(features & Corrade::Cpu::Avx512f) \ + return function(Corrade::Cpu::Avx512f); \ + if(features & Corrade::Cpu::Avx2) \ + return function(Corrade::Cpu::Avx2); \ + if(features & Corrade::Cpu::Avx) \ + return function(Corrade::Cpu::Avx); \ + if(features & Corrade::Cpu::Sse42) \ + return function(Corrade::Cpu::Sse42); \ + if(features & Corrade::Cpu::Sse41) \ + return function(Corrade::Cpu::Sse41); \ + if(features & Corrade::Cpu::Ssse3) \ + return function(Corrade::Cpu::Ssse3); \ + if(features & Corrade::Cpu::Sse3) \ + return function(Corrade::Cpu::Sse3); \ + if(features & Corrade::Cpu::Sse2) \ + return function(Corrade::Cpu::Sse2); \ + return function(Corrade::Cpu::Scalar); \ + } +#elif defined(CORRADE_TARGET_ARM) +#define CORRADE_CPU_DISPATCHER_BASE(function) \ + decltype(function(Corrade::Cpu::Scalar)) function(Corrade::Cpu::Features features) { \ + if(features & Corrade::Cpu::NeonFp16) \ + return function(Corrade::Cpu::NeonFp16); \ + if(features & Corrade::Cpu::NeonFma) \ + return function(Corrade::Cpu::NeonFma); \ + if(features & Corrade::Cpu::Neon) \ + return function(Corrade::Cpu::Neon); \ + return function(Corrade::Cpu::Scalar); \ + } +#elif defined(CORRADE_TARGET_WASM) +#define CORRADE_CPU_DISPATCHER_BASE(function) \ + decltype(function(Corrade::Cpu::Scalar)) function(Corrade::Cpu::Features features) { \ + if(features & Corrade::Cpu::Simd128) \ + return function(Corrade::Cpu::Simd128); \ + return function(Corrade::Cpu::Scalar); \ + } +#else +#define CORRADE_CPU_DISPATCHER_BASE(function) \ + decltype(function(Corrade::Cpu::Scalar)) function(Corrade::Cpu::Features features) { \ + return function(Corrade::Cpu::Scalar); \ + } +#endif + +#ifndef DOXYGEN_GENERATING_OUTPUT +#ifdef CORRADE_TARGET_X86 +#define _CORRADE_CPU_DISPATCHER_IMPLEMENTATION(function, extra) \ + if(features >= (Corrade::Cpu::Avx512f extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Avx512f extra)); \ + if(features >= (Corrade::Cpu::Avx2 extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Avx2 extra)); \ + if(features >= (Corrade::Cpu::Avx extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Avx extra)); \ + if(features >= (Corrade::Cpu::Sse42 extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Sse42 extra)); \ + if(features >= (Corrade::Cpu::Sse41 extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Sse41 extra)); \ + if(features >= (Corrade::Cpu::Ssse3 extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Ssse3 extra)); \ + if(features >= (Corrade::Cpu::Sse3 extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Sse3 extra)); \ + if(features >= (Corrade::Cpu::Sse2 extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Sse2 extra)); \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar extra)); +#elif defined(CORRADE_TARGET_ARM) +#define _CORRADE_CPU_DISPATCHER_IMPLEMENTATION(function, extra) \ + if(features >= (Corrade::Cpu::NeonFp16 extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::NeonFp16 extra)); \ + if(features >= (Corrade::Cpu::NeonFma extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::NeonFma extra)); \ + if(features >= (Corrade::Cpu::Neon extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Neon extra)); \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar extra)); +#elif defined(CORRADE_TARGET_WASM) +#define _CORRADE_CPU_DISPATCHER_IMPLEMENTATION(function, extra) \ + if(features >= (Corrade::Cpu::Simd128 extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Simd128 extra)); \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar extra)); +#else +#define _CORRADE_CPU_DISPATCHER_IMPLEMENTATION(function, extra) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar extra)); +#endif + +/* CORRADE_CPU_DISPATCHER() specialization for 0 extra instruction sets. + Basically equivalent to CORRADE_CPU_DISPATCHER_BASE() except for the extra + CORRADE_CPU_SELECT() macro. */ +#define _CORRADE_CPU_DISPATCHER0(function) \ + decltype(function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar))) function(Corrade::Cpu::Features features) { \ + _CORRADE_CPU_DISPATCHER_IMPLEMENTATION(function, ) \ + } + +/* CORRADE_CPU_DISPATCHER() specialization for 1+ extra instruction sets. On + Clang this still generates quite a reasonable code for 2 extra sets compared + to CORRADE_CPU_DISPATCHER_BASE(), on GCC it's ~25% longer but also still + reasonable. I attempted adding an "unrolled" _CORRADE_CPU_DISPATCHER2() but + it made everything significantly worse on both compilers, more than doubling + the amount of generated code. */ +#define _CORRADE_CPU_DISPATCHERn(function, ...) \ + template CORRADE_ALWAYS_INLINE decltype(function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar))) function ## Internal(Corrade::Cpu::Features features, Corrade::Cpu::Implementation::Tags) { \ + _CORRADE_CPU_DISPATCHER_IMPLEMENTATION(function, |Corrade::Cpu::Implementation::Tags{Corrade::Cpu::Implementation::Init}) \ + } \ + template CORRADE_ALWAYS_INLINE decltype(function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar))) function ## Internal(Corrade::Cpu::Features features, Corrade::Cpu::Implementation::Tags extra, First first, Next... next) { \ + static_assert(!(static_cast(Corrade::Cpu::Implementation::tags(First{Corrade::Cpu::Implementation::Init})) & Corrade::Cpu::Implementation::BaseTagMask), \ + "only extra instruction set tags should be explicitly listed"); \ + if(features & first) \ + return function ## Internal(features, extra|first, next...); \ + else \ + return function ## Internal(features, extra, next...); \ + } \ + decltype(function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar))) function(Corrade::Cpu::Features features) { \ + return function ## Internal(features, Corrade::Cpu::Implementation::Tags<0>{Corrade::Cpu::Implementation::Init}, __VA_ARGS__); \ + } +#endif + +/** +@brief Create a function for a runtime dispatch on a base CPU instruction set and select extra instruction sets +@m_since_latest + +Given a set of function overloads named @p function that accept a CPU tag +combination wrapped in @ref CORRADE_CPU_DECLARE() as a parameter, all returning +a function pointer of the same type, creates a function with signature +@cpp function(Cpu::Features) @ce which will select among the overloads using a +runtime-specified @relativeref{Corrade,Cpu::Features}, using the same rules as +the compile-time overload selection. The extra instruction sets considered in +the overload selection are specified as additional parameters to the macro, +specifying none is valid as well. For this macro to work, at the very least +there has to be an overload with a +@ref Corrade::Cpu::Scalar "CORRADE_CPU_DECLARE(Cpu::Scalar)" argument. See +@ref Cpu-usage-automatic-runtime-dispatch for more information and an example. + +For a dispatch using just the base instruction set use +@ref CORRADE_CPU_DISPATCHER_BASE() instead. +*/ +#ifdef DOXYGEN_GENERATING_OUTPUT +#define CORRADE_CPU_DISPATCHER(function, ...) +#elif !defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_CLANG_CL) +#define CORRADE_CPU_DISPATCHER(...) \ + _CORRADE_HELPER_PICK(__VA_ARGS__, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHER0, )(__VA_ARGS__) +#else +/* Workaround for MSVC not being able to expand __VA_ARGS__ correctly. Would + work with /Zc:preprocessor or /experimental:preprocessor, but I'm not + enabling that globally yet. Source: https://stackoverflow.com/a/5134656 */ +#define _CORRADE_CPU_DISPATCHER_FFS_MSVC_EXPAND_THIS(x) x +#define CORRADE_CPU_DISPATCHER(...) \ + _CORRADE_CPU_DISPATCHER_FFS_MSVC_EXPAND_THIS( _CORRADE_HELPER_PICK(__VA_ARGS__, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHER0, )(__VA_ARGS__)) +#endif + +/** +@brief Create a runtime-dispatched function pointer +@m_since_latest + +Assuming a @p dispatcher was defined with either @ref CORRADE_CPU_DISPATCHER() +or @ref CORRADE_CPU_DISPATCHER_BASE(), defines a function pointer variable with +a signature specified in the second variadic argument. In a global constructor +the variable is assigned a function pointer returned by @p dispatcher for +@relativeref{Corrade,Cpu::runtimeFeatures()}. + +The pointer can be changed afterwards, such as for testing purposes, See also +@ref CORRADE_CPU_DISPATCHED_IFUNC() which avoids the overhead of function +pointer indirection. + +See @ref Cpu-usage-automatic-cached-dispatch for more information, usage +example and overhead comparison. +*/ +#define CORRADE_CPU_DISPATCHED_POINTER(dispatcher, ...) \ + __VA_ARGS__ = dispatcher(Corrade::Cpu::runtimeFeatures()); + +/** +@brief Create a runtime-dispatched function via GNU IFUNC +@m_since_latest + +Available only if @ref CORRADE_CPU_USE_IFUNC is enabled. Assuming a +@p dispatcher was defined with either @ref CORRADE_CPU_DISPATCHER() or +@ref CORRADE_CPU_DISPATCHER_BASE(), defines a function with a signature +specified via the third variadic argument. The signature has to match @p type. +The function uses the [GNU IFUNC](https://sourceware.org/glibc/wiki/GNU_IFUNC) +mechanism, which causes the function call to be resolved to a function pointer +returned by @p dispatcher for @relativeref{Corrade,Cpu::runtimeFeatures()}. The +dispatch is performed by the dynamic linker during early startup and cannot be +changed afterwards. + +If @ref CORRADE_CPU_USE_IFUNC isn't available, is explicitly disabled or if +you need to be able to subsequently change the dispatched-to function (such as +for testing purposes), use @ref CORRADE_CPU_DISPATCHED_POINTER() instead. + +See @ref Cpu-usage-automatic-cached-dispatch for more information, usage +example and overhead comparison. +*/ +#if defined(CORRADE_CPU_USE_IFUNC) || defined(DOXYGEN_GENERATING_OUTPUT) +/* On ARM we get CPU features through getauxval() but it can't be called from + an ifunc resolver because it's too early at that point. Instead, AT_HWCAPS + is passed to it from outside, so there we call an internal variant with the + caps parameter -- see its documentation in Cpu.cpp for more info. On x86 + calling into CPUID from within an ifunc resolver is no problem. + + Although not specifically documented anywhere, the dispatcher has to have + C++ mangling disabled in order to be found by __attribute__((ifunc)), on + both GCC and Clang. That however means it's exported even if inside an + anonymous namespace, which is undesirable. To fix that, it's marked as + static... */ +#ifdef CORRADE_TARGET_CLANG +/* ... unfortunately the static makes Clang not find the name again, so there + we can't use it. But, to drown even deeper, not using the static causes the + -Wmissing-prototypes macro to get fired (which is enabled globally because + it has obvious benefits), so to avoid noise it has to be disabled here. */ +#ifndef CORRADE_TARGET_ARM +#define CORRADE_CPU_DISPATCHED_IFUNC(dispatcher, ...) \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wmissing-prototypes\"") \ + extern "C" decltype(dispatcher(std::declval())) dispatcher() { \ + return dispatcher(Corrade::Cpu::runtimeFeatures()); \ + } \ + __VA_ARGS__ __attribute__((ifunc(#dispatcher))); \ + _Pragma("GCC diagnostic pop") +#else +#define CORRADE_CPU_DISPATCHED_IFUNC(dispatcher, ...) \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wmissing-prototypes\"") \ + extern "C" decltype(dispatcher(std::declval())) dispatcher(unsigned long caps) { \ + return dispatcher(Corrade::Cpu::Implementation::runtimeFeatures(caps)); \ + } \ + __VA_ARGS__ __attribute__((ifunc(#dispatcher))); \ + _Pragma("GCC diagnostic pop") +#endif +#elif defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ < 409 +/* Furthermore, due to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58105 + the resolver function won't work on GCC 4.8 unless it's marked with + CORRADE_NEVER_INLINE. That, however, causes GCC 4.8 to spit out a bogus + warning that the dispatcher function is redeclared as noinline -- it thinks + it's the same function as the always-inline lambda wrappers which have the + same name. Despite the warning it works, but to avoid useless noise the + ifunc dispatcher is named differently. */ +#ifndef CORRADE_TARGET_ARM +#define CORRADE_CPU_DISPATCHED_IFUNC(dispatcher, ...) \ + extern "C" { CORRADE_NEVER_INLINE static decltype(dispatcher(std::declval())) dispatcher ## Ifunc() { \ + return dispatcher(Corrade::Cpu::runtimeFeatures()); \ + }} \ + __VA_ARGS__ __attribute__((ifunc(#dispatcher "Ifunc"))); +#else +#define CORRADE_CPU_DISPATCHED_IFUNC(dispatcher, ...) \ + extern "C" { CORRADE_NEVER_INLINE static decltype(dispatcher(std::declval())) dispatcher ## Ifunc(unsigned long caps) { \ + return dispatcher(Corrade::Cpu::Implementation::runtimeFeatures(caps)); \ + }} \ + __VA_ARGS__ __attribute__((ifunc(#dispatcher "Ifunc"))); +#endif +#else +/* Only GCC 4.9+ has the implementation in the most minimal form. */ +#ifndef CORRADE_TARGET_ARM +#define CORRADE_CPU_DISPATCHED_IFUNC(dispatcher, ...) \ + extern "C" { static decltype(dispatcher(std::declval())) dispatcher() { \ + return dispatcher(Corrade::Cpu::runtimeFeatures()); \ + }} \ + __VA_ARGS__ __attribute__((ifunc(#dispatcher))); +#else +#define CORRADE_CPU_DISPATCHED_IFUNC(dispatcher, ...) \ + extern "C" { static decltype(dispatcher(std::declval())) dispatcher(unsigned long caps) { \ + return dispatcher(Corrade::Cpu::Implementation::runtimeFeatures(caps)); \ + }} \ + __VA_ARGS__ __attribute__((ifunc(#dispatcher))); +#endif +#endif +#endif + +#if defined(CORRADE_TARGET_X86) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief Enable SSE2 for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC, Clang and @ref CORRADE_TARGET_CLANG_CL "clang-cl" +expands to @cpp __attribute__((__target__("sse2"))) @ce, allowing use of +[SSE2](https://en.wikipedia.org/wiki/SSE2) and earlier SSE instructions inside +a function annotated with this macro without having to specify `-msse2` for the +whole compilation unit. On x86 MSVC expands to nothing, as the compiler doesn't +restrict use of intrinsics in any way. Not defined on other compilers or +architectures. + +As a special case, if @ref CORRADE_TARGET_SSE2 is present (meaning SSE2 is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Implied by @ref CORRADE_ENABLE_SSE3. See @ref Cpu-usage-target-attributes for +more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsSse2.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::Sse2}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_SSE2) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_SSE2 +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_SSE2 +#endif +#elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) +#define CORRADE_ENABLE_SSE2 __attribute__((__target__("sse2"))) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_SSE2 "sse2", +#endif +#elif defined(CORRADE_TARGET_MSVC) +#define CORRADE_ENABLE_SSE2 +#endif + +/** +@brief Enable SSE3 for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to +@cpp __attribute__((__target__("sse3"))) @ce, allowing use of +[SSE3](https://en.wikipedia.org/wiki/SSE3) and earlier SSE intrinsics inside a +function annotated with this macro without having to specify `-msse3` for the +whole compilation unit. On x86 MSVC expands to nothing, as the compiler doesn't +restrict use of intrinsics in any way. Not defined on other compilers or +architectures. + +As a special case, if @ref CORRADE_TARGET_SSE3 is present (meaning SSE3 is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Superset of @ref CORRADE_ENABLE_SSE2, implied by @ref CORRADE_ENABLE_SSSE3. See +@ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsSse3.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::Sse3}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_SSE3) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_SSE3 +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_SSE3 +#endif +#elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) +/* The -msse3 option implies -msse2 on both GCC and Clang, so no need to + specify those as well (verified with `echo | gcc -dM -E - -msse3`) */ +#define CORRADE_ENABLE_SSE3 __attribute__((__target__("sse3"))) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_SSE3 "sse3", +#endif +#elif defined(CORRADE_TARGET_MSVC) +#define CORRADE_ENABLE_SSE3 +#endif + +/** +@brief Enable SSSE3 for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC, Clang and @ref CORRADE_TARGET_CLANG_CL "clang-cl" +expands to @cpp __attribute__((__target__("ssse3"))) @ce, allowing use of +[SSSE3](https://en.wikipedia.org/wiki/SSSE3) and earlier SSE instructions +inside a function annotated with this macro without having to specify `-mssse3` +for the whole compilation unit. On x86 MSVC expands to nothing, as the compiler +doesn't restrict use of intrinsics in any way. Not defined on other compilers +or architectures. + +As a special case, if @ref CORRADE_TARGET_SSSE3 is present (meaning SSSE3 is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Superset of @ref CORRADE_ENABLE_SSE3, implied by @ref CORRADE_ENABLE_SSE41. See +@ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsSsse3.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::Ssse3}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_SSSE3) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_SSSE3 +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_SSSE3 +#endif +#elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) +/* The -mssse3 option implies -msse2 -msse3 on both GCC and Clang, so no need + to specify those as well (verified with `echo | gcc -dM -E - -mssse3`) */ +#define CORRADE_ENABLE_SSSE3 __attribute__((__target__("ssse3"))) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_SSSE3 "ssse3", +#endif +#elif defined(CORRADE_TARGET_MSVC) +#define CORRADE_ENABLE_SSSE3 +#endif + +/** +@brief Enable SSE4.1 for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC, Clang and @ref CORRADE_TARGET_CLANG_CL "clang-cl" +expands to @cpp __attribute__((__target__("sse4.1"))) @ce, allowing use of +[SSE4.1](https://en.wikipedia.org/wiki/SSE4#SSE4.1) and earlier SSE +instructions inside a function annotated with this macro without having to +specify `-msse4.1` for the whole compilation unit. On x86 MSVC expands to +nothing, as the compiler doesn't restrict use of intrinsics in any way. Not +defined on other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_SSE41 is present (meaning SSE4.1 is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Superset of @ref CORRADE_ENABLE_SSSE3, implied by @ref CORRADE_ENABLE_SSE42. +See @ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-danger} + +@par + Unless @ref CORRADE_TARGET_SSE41 is present, this macro is not defined on + GCC 4.8, as SSE4.1 intrinsics only work if `-msse4.2` is specified as well + due to both SSE4.1 and 4.2 intrinsics living in the same header. You can + only use @ref CORRADE_ENABLE_SSE42 in this case. + +@see @relativeref{Corrade,Cpu::Sse41}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_SSE41) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_SSE41 +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_SSE41 +#endif +#elif (defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ >= 409) || defined(CORRADE_TARGET_CLANG) /* also matches clang-cl */ +/* The -msse4.1 option implies -msse2 -msse3 -mssse3 on both GCC and Clang, so + no need to specify those as well (verified with + `echo | gcc -dM -E - -msse4.1`) */ +#define CORRADE_ENABLE_SSE41 __attribute__((__target__("sse4.1"))) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_SSE41 "sse4.1", +#endif +#elif defined(CORRADE_TARGET_MSVC) +#define CORRADE_ENABLE_SSE41 +#endif + +/** +@brief Enable SSE4.2 for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC, Clang and @ref CORRADE_TARGET_CLANG_CL "clang-cl" +expands to @cpp __attribute__((__target__("sse4.2"))) @ce, allowing use of +[SSE4.2](https://en.wikipedia.org/wiki/SSE4#SSE4.2) and earlier SSE +instructions inside a function annotated with this macro without having to +specify `-msse4.2` for the whole compilation unit. On x86 MSVC expands to +nothing, as the compiler doesn't restrict use of intrinsics in any way. Not +defined on other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_SSE42 is defined (meaning SSE4.2 is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Superset of @ref CORRADE_ENABLE_SSE41, implied by @ref CORRADE_ENABLE_AVX. See +@ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsSse4.h instead of + @cpp #include @ce and @cpp #include @ce to be + able to access the intrinsics on this compiler. + +@see @relativeref{Corrade,Cpu::Sse42}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_SSE42) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_SSE42 +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_SSE42 +#endif +#elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) +/* The -msse4.2 option implies -msse2 -msse3 -mssse3 -msse4.1 on both GCC and + Clang, so no need to specify those as well (verified with + `echo | gcc -dM -E - -msse4.2`) */ +#define CORRADE_ENABLE_SSE42 __attribute__((__target__("sse4.2"))) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_SSE42 "sse4.2", +#endif +#elif defined(CORRADE_TARGET_MSVC) +#define CORRADE_ENABLE_SSE42 +#endif + +/** +@brief Enable POPCNT for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC, Clang and @ref CORRADE_TARGET_CLANG_CL "clang-cl" +expands to @cpp __attribute__((__target__("popcnt"))) @ce, allowing use of the +[POPCNT](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation)) +instructions inside a function annotated with this macro without having to +specify `-mpopcnt` for the whole compilation unit. On x86 MSVC expands to +nothing, as the compiler doesn't restrict use of intrinsics in any way. Not +defined on GCC 4.8, as there it's not generally possible to enable it alongside +other instruction sets without running into linker errors. Not defined on other +compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_POPCNT is defined (meaning POCNT is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Neither a superset nor implied by any other `CORRADE_ENABLE_*` macro, so you +may need to specify it together with others. See +@ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8 or Clang < 7, you may also want to use + @ref Corrade/Utility/IntrinsicsSse4.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::Popcnt}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_POPCNT) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_POPCNT +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_POPCNT +#endif +#elif (defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ >= 409) || defined(CORRADE_TARGET_CLANG) /* matches clang-cl */ +#define CORRADE_ENABLE_POPCNT __attribute__((__target__("popcnt"))) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_POPCNT "popcnt", +#endif +#elif defined(CORRADE_TARGET_MSVC) +#define CORRADE_ENABLE_POPCNT +#endif + +/** +@brief Enable LZCNT for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to +@cpp __attribute__((__target__("lzcnt"))) @ce, allowing use of the +[LZCNT](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation)) +instructions inside a function annotated with this macro without having to +specify `-mlzcnt` for the whole compilation unit. On x86 MSVC expands to +nothing, as the compiler doesn't restrict use of intrinsics in any way. Unlike +the SSE variants and POPCNT this macro is not defined on +@ref CORRADE_TARGET_CLANG_CL "clang-cl", as there LZCNT, BMI1, AVX and newer +intrinsics are provided only if enabled on compiler command line. Not defined +on GCC 4.8, as there it's not generally possible to enable it alongside +unrelated instruction sets without running into linker errors. Not defined on +other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_LZCNT is defined (meaning LZCNT is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Neither a superset nor implied by any other `CORRADE_ENABLE_*` macro, so you +may need to specify it together with others. See +@ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsAvx.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::Lzcnt}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_LZCNT) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_LZCNT +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_LZCNT +#endif +#elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) /* does not match clang-cl */ +#define CORRADE_ENABLE_LZCNT __attribute__((__target__("lzcnt"))) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_LZCNT "lzcnt", +#endif +/* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, + still present in Jul 2022, meaning we can only use these if __LZCNT__ is + defined. Funnily enough the older headers don't have this on their own, only + . Also I don't think "Actually using intrinsics on Windows + already requires the right /arch: settings" is correct. */ +#elif defined(CORRADE_TARGET_MSVC) && !defined(CORRADE_TARGET_CLANG_CL) +#define CORRADE_ENABLE_LZCNT +#endif + +/** +@brief Enable BMI1 for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC, Clang expands to +@cpp __attribute__((__target__("bmi"))) @ce, allowing use of the +[BMI1](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#BMI1_(Bit_Manipulation_Instruction_Set_1)) +instructions inside a function annotated with this macro without having to +specify `-mbmi` for the whole compilation unit. On x86 MSVC expands to nothing, +as the compiler doesn't restrict use of intrinsics in any way. Unlike the SSE +variants and POPCNT this macro is not defined on +@ref CORRADE_TARGET_CLANG_CL "clang-cl", as there LZCNT, BMI1, AVX and newer +intrinsics are provided only if enabled on compiler command line. Not defined +on GCC 4.8, as there it's not generally possible to enable it alongside +unrelated instruction sets without running into linker errors. Not defined on +other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_BMI1 is defined (meaning BMI1 is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Neither a superset nor implied by any other `CORRADE_ENABLE_*` macro, so you +may need to specify it together with others. See +@ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsAvx.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::Bmi1}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_BMI1) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_BMI1 +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_BMI1 +#endif +#elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) /* does not match clang-cl */ +#define CORRADE_ENABLE_BMI1 __attribute__((__target__("bmi"))) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_BMI1 "bmi", +#endif +/* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, + still present in Jul 2022, meaning we can only use these if __BMI__ is + defined. Funnily enough the older headers don't have this on their own, only + . Also I don't think "Actually using intrinsics on Windows + already requires the right /arch: settings" is correct. */ +#elif defined(CORRADE_TARGET_MSVC) && !defined(CORRADE_TARGET_CLANG_CL) +#define CORRADE_ENABLE_BMI1 +#endif + +/** +@brief Enable AVX for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to +@cpp __attribute__((__target__("avx"))) @ce, allowing use of +[AVX](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions) and all earlier +SSE instructions inside a function annotated with this macro without having to +specify `-mavx` for the whole compilation unit. On x86 MSVC expands to nothing, +as the compiler doesn't restrict use of intrinsics in any way. Unlike the SSE +variants this macro is not defined on @ref CORRADE_TARGET_CLANG_CL "clang-cl", +as there AVX and newer intrinsics are provided only if enabled on compiler +command line. Not defined on other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_AVX is present (meaning AVX is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Superset of @ref CORRADE_ENABLE_SSE42, implied by @ref CORRADE_ENABLE_AVX2. See +@ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsAvx.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::Avx}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_AVX) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_AVX +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_AVX +#endif +#elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ +/* The -mavx option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 on both GCC + and Clang, so no need to specify those as well (verified with + `echo | gcc -dM -E - -mavx`) */ +#define CORRADE_ENABLE_AVX __attribute__((__target__("avx"))) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_AVX "avx", +#endif +/* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, + still present in Jul 2022, meaning we can only use these if __AVX__ is + defined. Funnily enough the older headers don't have this on their own, only + . Also I don't think "Actually using intrinsics on Windows + already requires the right /arch: settings" is correct. */ +#elif defined(CORRADE_TARGET_MSVC) && !defined(CORRADE_TARGET_CLANG_CL) +#define CORRADE_ENABLE_AVX +#endif + +/** +@brief Enable AVX F16C for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to +@cpp __attribute__((__target__("f16c"))) @ce, allowing use of +[F16C](https://en.wikipedia.org/wiki/F16C) instructions inside a function +annotated with this macro without having to specify `-mf16c` for the whole +compilation unit. On x86 MSVC expands to nothing, as the compiler doesn't +restrict use of intrinsics in any way. Unlike the SSE variants this macro is +not defined on @ref CORRADE_TARGET_CLANG_CL "clang-cl", as there AVX and newer +intrinsics are provided only if enabled on compiler command line. Not defined +on GCC 4.8, as there it's not generally possible to enable it alongside other +instruction sets without running into linker errors. Not defined on other +compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_AVX_F16C is present (meaning AVX F16C +is enabled for the whole compilation unit), this macro is defined as empty on +all compilers. + +Superset of @ref CORRADE_ENABLE_AVX on both GCC and Clang. However not +portably implied by any other `CORRADE_ENABLE_*` macro so you may need to +specify it together with others. See @ref Cpu-usage-target-attributes for more +information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsAvx.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::AvxF16c}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_AVX_F16C) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_AVX_F16C +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_AVX_F16C +#endif +#elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) /* does not match clang-cl */ +/* The -mf16c option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx on + both GCC and Clang (verified with `echo | gcc -dM -E - -mf16c`) */ +#define CORRADE_ENABLE_AVX_F16C __attribute__((__target__("f16c"))) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_AVX_F16C "f16c", +#endif +/* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, + still present in Jul 2022, meaning we can only use these if __F16C__ is + defined. Funnily enough the older headers don't have this on their own, only + . Also I don't think "Actually using intrinsics on Windows + already requires the right /arch: settings" is correct. */ +#elif defined(CORRADE_TARGET_MSVC) && !defined(CORRADE_TARGET_CLANG_CL) +#define CORRADE_ENABLE_AVX_F16C +#endif + +/** +@brief Enable AVX FMA for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to +@cpp __attribute__((__target__("fma"))) @ce, allowing use of +[FMA](https://en.wikipedia.org/wiki/FMA_instruction_set) instructions inside a +function annotated with this macro without having to specify `-mfma` for the +whole compilation unit. On x86 MSVC expands to nothing, as the compiler doesn't +restrict use of intrinsics in any way. Unlike the SSE variants this macro is +not defined on @ref CORRADE_TARGET_CLANG_CL "clang-cl", as there AVX and newer +intrinsics are provided only if enabled on compiler command line. Not defined +on GCC 4.8, as there it's not generally possible to enable it alongside other +instruction sets without running into linker errors. Not defined on other +compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_AVX_FMA is present (meaning AVX with +FMA is enabled for the whole compilation unit), this macro is defined as empty +on all compilers. + +Superset of @ref CORRADE_ENABLE_AVX on both GCC and Clang. However not +portably implied by any other `CORRADE_ENABLE_*` macro so you may need to +specify it together with others. See @ref Cpu-usage-target-attributes for more +information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsAvx.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::AvxFma}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_AVX_FMA) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_AVX_FMA +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_AVX_FMA +#endif +#elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) /* does not match clang-cl */ +/* The -mfma option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx on + both GCC and Clang (verified with `echo | gcc -dM -E - -mf16c`) */ +#define CORRADE_ENABLE_AVX_FMA __attribute__((__target__("fma"))) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_AVX_FMA "fma", +#endif +/* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, + still present in Jul 2022, meaning we can only use these if __FMA__ is + defined. Funnily enough the older headers don't have this on their own, only + . Also I don't think "Actually using intrinsics on Windows + already requires the right /arch: settings" is correct. */ +#elif defined(CORRADE_TARGET_MSVC) && !defined(CORRADE_TARGET_CLANG_CL) +#define CORRADE_ENABLE_AVX_FMA +#endif + +/** +@brief Enable AVX2 for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to +@cpp __attribute__((__target__("avx2"))) @ce, allowing use of +[AVX2](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#Advanced_Vector_Extensions_2), +FMA, F16C, AVX and all earlier SSE instructions inside a function annotated +with this macro without having to specify `-mavx2` for the whole compilation +unit. On x86 MSVC expands to nothing, as the compiler doesn't restrict use of +intrinsics in any way. Unlike the SSE variants this macro is not defined on +@ref CORRADE_TARGET_CLANG_CL "clang-cl", as there AVX and newer intrinsics are +provided only if enabled on compiler command line. Not defined on other +compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_AVX2 is present (meaning AVX2 is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Superset of @ref CORRADE_ENABLE_AVX, implied by @ref CORRADE_ENABLE_AVX512F. +See @ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsAvx.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::Avx2}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_AVX2) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_AVX2 +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_AVX2 +#endif +#elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ +/* The -mavx2 option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx on + both GCC and Clang, so no need to specify those as well (verified with + `echo | gcc -dM -E - -mavx2`) */ +#define CORRADE_ENABLE_AVX2 __attribute__((__target__("avx2"))) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_AVX2 "avx2", +#endif +/* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, + still present in Jul 2022, meaning we can only use these if __AVX2__ is + defined. Funnily enough the older headers don't have this on their own, only + . Also I don't think "Actually using intrinsics on Windows + already requires the right /arch: settings" is correct. */ +#elif defined(CORRADE_TARGET_MSVC) && !defined(CORRADE_TARGET_CLANG_CL) +#define CORRADE_ENABLE_AVX2 +#endif + +/** +@brief Enable AVX-512 Foundation for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC 4.9+ and Clang expands to +@cpp __attribute__((__target__("avx512f"))) @ce, allowing use of +[AVX-512](https://en.wikipedia.org/wiki/AVX-512) Foundation and all earlier AVX +and SSE instructions inside a function annotated with this macro without having +to specify `-mavx512f` for the whole compilation unit. On x86 MSVC 2017 15.3+ +expands to nothing, as the compiler doesn't restrict use of intrinsics in any +way. Unlike the SSE variants this macro is not defined on +@ref CORRADE_TARGET_CLANG_CL "clang-cl", as there AVX and newer intrinsics are +provided only if enabled on compiler command line. Not defined on other +compilers, earlier compiler versions without AVX-512 support or other +architectures. + +As a special case, if @ref CORRADE_TARGET_AVX512F is present (meaning AVX-512 +Foundation is enabled for the whole compilation unit), this macro is defined as +empty on all compilers. + +Superset of @ref CORRADE_ENABLE_AVX2. See @ref Cpu-usage-target-attributes for +more information and usage example. +@see @relativeref{Corrade,Cpu::Avx512f}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_AVX512F) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_AVX512F +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_AVX512F +#endif +#elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) /* does not match clang-cl */ +/* The -mavx512 option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx + -mavx2 on both GCC and Clang, so no need to specify those as well (verified + with `echo | gcc -dM -E - -mavx512f`) */ +#define CORRADE_ENABLE_AVX512F __attribute__((__target__("avx512f"))) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_AVX512F "avx512f", +#endif +/* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, + still present in Jul 2022, meaning we can only use these if __AVX512F__ is + defined. Funnily enough the older headers don't have this on their own, only + . Also I don't think "Actually using intrinsics on Windows + already requires the right /arch: settings" is correct. */ +#elif defined(CORRADE_TARGET_MSVC) && _MSC_VER >= 1911 && !defined(CORRADE_TARGET_CLANG_CL) +#define CORRADE_ENABLE_AVX512F +#endif +#endif + +#if defined(CORRADE_TARGET_ARM) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief Enable NEON for given function +@m_since_latest + +On 32-bit @ref CORRADE_TARGET_ARM "ARM" GCC expands to +@cpp __attribute__((__target__("fpu=neon"))) @ce, allowing use of +[NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)) +instructions inside a function annotated with this macro without having to +specify `-mfpu=neon` for the whole compilation unit. On ARM MSVC expands to +nothing, as the compiler doesn't restrict use of intrinsics in any way. In +contrast to GCC, this macro is not defined on Clang, as it makes the NEON +intrinsics available only if enabled on compiler command line. Not defined on +other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_NEON is present (meaning NEON is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. This is also the case for ARM64, where NEON support is implicit +(and where `-mfpu=neon` is unrecognized). + +Implied by @ref CORRADE_ENABLE_NEON_FMA. See @ref Cpu-usage-target-attributes +for more information and usage example. +@see @relativeref{Corrade,Cpu::Neon}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_NEON) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_NEON +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_NEON +#endif +/* https://github.com/android/ndk/issues/1066 is the only reported (and + ignored) issue I found, feels strange that people would just not use ifunc + or target attributes on Android at all and instead put everything in + separate files. Needs further investigation. Too bad most ARM platforms + ditched GCC, where this works properly. */ +#elif defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) +#define CORRADE_ENABLE_NEON __attribute__((__target__("fpu=neon"))) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_NEON "fpu=neon", +#endif +#elif defined(CORRADE_TARGET_MSVC) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_NEON +#endif + +/** +@brief Enable NEON FMA for given function +@m_since_latest + +On 32-bit @ref CORRADE_TARGET_ARM "ARM" GCC expands to +@cpp __attribute__((__target__("fpu=neon-vfpv4"))) @ce, allowing use of +[NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)) FMA +instructions inside a function annotated with this macro without having to +specify `-mfpu=neon-vfpv4` for the whole compilation unit. On ARM MSVC expands +to nothing, as the compiler doesn't restrict use of intrinsics in any way. In +contrast to GCC, this macro is not defined on Clang, as it makes the NEON FMA +intrinsics available only if enabled on compiler command line. Not defined on +other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_NEON_FMA is present (meaning NEON FMA +is enabled for the whole compilation unit), this macro is defined as empty on +all compilers. This is also the case for ARM64, where NEON support is implicit +(and where `-mfpu=neon-vfpv4` is unrecognized). + +Superset of @ref CORRADE_ENABLE_NEON, implied by @ref CORRADE_ENABLE_NEON_FP16. +See @ref Cpu-usage-target-attributes for more information and usage example. +@see @relativeref{Corrade,Cpu::NeonFma}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_NEON_FMA) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_NEON_FMA +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_NEON_FMA +#endif +/* See CORRADE_ENABLE_NEON above for details about Clang */ +#elif defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) +#define CORRADE_ENABLE_NEON_FMA __attribute__((__target__("fpu=neon-vfpv4"))) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_NEON_FMA "fpu=neon-vfpv4", +#endif +#elif defined(CORRADE_TARGET_MSVC) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_NEON_FMA +#endif + +/** +@brief Enable NEON FP16 for given function +@m_since_latest + +On @ref CORRADE_TARGET_ARM "ARM" GCC expands to +@cpp __attribute__((__target__("arch=armv8.2-a+fp16"))) @ce, allowing use of +ARMv8.2-a [NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)) +FP16 vector arithmetic inside a function annotated with this macro without +having to specify `-march=armv8.2-a+fp16` for the whole compilation unit. On +ARM MSVC expands to nothing, as the compiler doesn't restrict use of intrinsics +in any way. In contrast to GCC, this macro is not defined on Clang, as it makes +the NEON FP16 intrinsics available only if enabled on compiler command line. +Not defined on other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_NEON_FP16 is present (meaning NEON +FP16 is enabled for the whole compilation unit), this macro is defined as empty +on all compilers. + +Superset of @ref CORRADE_ENABLE_NEON_FMA. See @ref Cpu-usage-target-attributes +for more information and usage example. +@see @relativeref{Corrade,Cpu::NeonFp16}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_NEON_FP16) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_NEON_FP16 +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_NEON_FP16 +#endif +/* See CORRADE_ENABLE_NEON above for details about Clang */ +#elif defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) +#define CORRADE_ENABLE_NEON_FP16 __attribute__((__target__("arch=armv8.2-a+fp16"))) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_NEON_FP16 "arch=armv8.2-a+fp16", +#endif +#elif defined(CORRADE_TARGET_MSVC) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_NEON_FP16 +#endif +#endif + +#if defined(CORRADE_TARGET_WASM) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief Enable SIMD128 for given function +@m_since_latest + +Given that it's currently not possible to selectively use +[128-bit SIMD](https://github.com/webassembly/simd) in a WebAssembly module +without causing a compilation error on runtimes that don't support it, this +macro is only defined if @ref CORRADE_TARGET_SIMD128 is present (meaning +SIMD128 is explicitly enabled for the whole compilation unit), and is always +empty, as @cpp __attribute__((__target__("simd128"))) @ce would be redundant +if `-msimd128` is passed on the command line. + +The situation may change once the +[feature detection proposal](https://github.com/WebAssembly/feature-detection/blob/main/proposals/feature-detection/Overview.md) +is implemented, but likely only for instruction sets building on top of this +one. + +See @ref Cpu-usage-target-attributes for more information and usage example. +@see @relativeref{Corrade,Cpu::Simd128}, @ref CORRADE_ENABLE() +*/ +#if defined(CORRADE_TARGET_SIMD128) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_SIMD128 +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_SIMD128 +#endif +#endif +#endif + +#ifndef DOXYGEN_GENERATING_OUTPUT +/* GCC and Clang before version 8 treat + __attribute__((target("foo"))) __attribute__((target("bar"))) + as if only "bar" was specified, thus it's not possible to just put several + CORRADE_ENABLE_ macros after each other. Instead, the only accepted form is + __attribute__((target("foo,bar"))). Fortunately, string literal + concatenation works here, thus with some extra macro trickery we can produce + __attribute__((target("foo" "," "bar"))). The pieces are _CORRADE_ENABLE_* + variants defined if and only if a corresponding CORRADE_ENABLE_* macro is + defined. These can however be empty, thus it's not possible to just join + them all with "," in between. Instead, the macros themselves have a trailing + comma after the string literal (thus "foo",), which causes the empty macros + be filtered out when passed one after another (without commas) from + _CORRADE_ENABLEn to _CORRADE_ENABLE_CONCATENATE(). */ +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) +#define _CORRADE_ENABLE_CONCATENATE0(unused) +#define _CORRADE_ENABLE_CONCATENATE1(v0, unused) \ + __attribute__((__target__(v0))) +#define _CORRADE_ENABLE_CONCATENATE2(v0, v1, unused) \ + __attribute__((__target__(v0 "," v1))) +#define _CORRADE_ENABLE_CONCATENATE3(v0, v1, v2, unused) \ + __attribute__((__target__(v0 "," v1 "," v2))) +#define _CORRADE_ENABLE_CONCATENATE4(v0, v1, v2, v3, unused) \ + __attribute__((__target__(v0 "," v1 "," v2 "," v3))) +#define _CORRADE_ENABLE_CONCATENATE5(v0, v1, v2, v3, v4, unused) \ + __attribute__((__target__(v0 "," v1 "," v2 "," v3 "," v4))) +#define _CORRADE_ENABLE_CONCATENATE6(v0, v1, v2, v3, v4, v5, unused) \ + __attribute__((__target__(v0 "," v1 "," v2 "," v3 "," v4 "," v5))) +#define _CORRADE_ENABLE_CONCATENATE7(v0, v1, v2, v3, v4, v5, v6, unused) \ + __attribute__((__target__(v0 "," v1 "," v2 "," v3 "," v4 "," v5 "," v6))) +#define _CORRADE_ENABLE_CONCATENATE(...) \ + _CORRADE_HELPER_PICK(__VA_ARGS__, _CORRADE_ENABLE_CONCATENATE7, _CORRADE_ENABLE_CONCATENATE6, _CORRADE_ENABLE_CONCATENATE5, _CORRADE_ENABLE_CONCATENATE4, _CORRADE_ENABLE_CONCATENATE3, _CORRADE_ENABLE_CONCATENATE2, _CORRADE_ENABLE_CONCATENATE1, _CORRADE_ENABLE_CONCATENATE0, )(__VA_ARGS__) +/* No _CORRADE_HELPER_PASTE() needed here, as there's enough other indirections + to make that work */ +#define _CORRADE_ENABLE1(v0) \ + _CORRADE_ENABLE_CONCATENATE( \ + _CORRADE_ENABLE_ ## v0 \ + ) +#define _CORRADE_ENABLE2(v0, v1) \ + _CORRADE_ENABLE_CONCATENATE( \ + _CORRADE_ENABLE_ ## v0 \ + _CORRADE_ENABLE_ ## v1 \ + ) +#define _CORRADE_ENABLE3(v0, v1, v2) \ + _CORRADE_ENABLE_CONCATENATE( \ + _CORRADE_ENABLE_ ## v0 \ + _CORRADE_ENABLE_ ## v1 \ + _CORRADE_ENABLE_ ## v2 \ + ) +#define _CORRADE_ENABLE4(v0, v1, v2, v3) \ + _CORRADE_ENABLE_CONCATENATE( \ + _CORRADE_ENABLE_ ## v0 \ + _CORRADE_ENABLE_ ## v1 \ + _CORRADE_ENABLE_ ## v2 \ + _CORRADE_ENABLE_ ## v3 \ + ) +#define _CORRADE_ENABLE5(v0, v1, v2, v3, v4) \ + _CORRADE_ENABLE_CONCATENATE( \ + _CORRADE_ENABLE_ ## v0 \ + _CORRADE_ENABLE_ ## v1 \ + _CORRADE_ENABLE_ ## v2 \ + _CORRADE_ENABLE_ ## v3 \ + _CORRADE_ENABLE_ ## v4 \ + ) +#define _CORRADE_ENABLE6(v0, v1, v2, v3, v4, v5) \ + _CORRADE_ENABLE_CONCATENATE( \ + _CORRADE_ENABLE_ ## v0 \ + _CORRADE_ENABLE_ ## v1 \ + _CORRADE_ENABLE_ ## v2 \ + _CORRADE_ENABLE_ ## v3 \ + _CORRADE_ENABLE_ ## v4 \ + _CORRADE_ENABLE_ ## v5 \ + ) +#define _CORRADE_ENABLE7(v0, v1, v2, v3, v4, v5, v6) \ + _CORRADE_ENABLE_CONCATENATE( \ + _CORRADE_ENABLE_ ## v0 \ + _CORRADE_ENABLE_ ## v1 \ + _CORRADE_ENABLE_ ## v2 \ + _CORRADE_ENABLE_ ## v3 \ + _CORRADE_ENABLE_ ## v4 \ + _CORRADE_ENABLE_ ## v5 \ + _CORRADE_ENABLE_ ## v6 \ + ) +/* None of this is needed for Clang, fortunately, so here the whole thing + expands to just CORRADE_ENABLE_FOO CORRADE_ENABLE_BAR. I hope GCC eventually + fixes this as well, so keeping both variants so I can drop the GCC-specific + one in the future. As another future-proof this also gets used for any + compilers other than MSVC. MSVC's preprocessor won't be able to perform the + delayed expansion so CORRADE_ENABLE() */ +#elif defined(CORRADE_TARGET_CLANG) || !defined(CORRADE_TARGET_MSVC) +/* Using _CORRADE_HELPER_PASTE2() instead of _CORRADE_HELPER_PASTE() here, as + that's enough to make that work and it's less work for the preprocessor. + Concatenating directly doesn't work, unlike in the above case for GCC. */ +#define _CORRADE_ENABLE1(v0) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v0) +#define _CORRADE_ENABLE2(v0, v1) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v0) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v1) +#define _CORRADE_ENABLE3(v0, v1, v2) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v0) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v1) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v2) +#define _CORRADE_ENABLE4(v0, v1, v2, v3) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v0) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v1) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v2) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v3) +#define _CORRADE_ENABLE5(v0, v1, v2, v3, v4) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v0) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v1) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v2) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v3) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v4) +#define _CORRADE_ENABLE6(v0, v1, v2, v3, v4, v5) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v0) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v1) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v2) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v3) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v4) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v5) +#define _CORRADE_ENABLE7(v0, v1, v2, v3, v4, v5, v6) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v0) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v1) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v2) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v3) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v4) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v5) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v6) +#define _CORRADE_ENABLE8(v0, v1, v2, v3, v4, v5, v6, v7) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v0) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v1) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v2) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v3) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v4) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v5) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v6) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v7) +#endif +#endif + +/** +@brief Enable multiple targets for given function +@m_since_latest + +Accepts a comma-separated list of `CORRADE_ENABLE_*` macro suffixes, +effectively enabling given combination. For the macro to work, all +`CORRADE_ENABLE_*` macros corresponding to the arguments have to be defined, +the common usage pattern is thus in combination with an @cpp #ifdef @ce. See +@ref Cpu-usage-target-attributes for more information and an example. + +When multiple `CORRADE_ENABLE_*` macros are specified one after another, Clang +before version 8 and GCC would pick only the last specified, ignoring the +others. There the macro expands into a single combined +@cpp __attribute__((__target__(...))) @ce attribute. For other compilers except +MSVC it's just a shorthand for multiple `CORRADE_ENABLE_*` macros one after +another. On MSVC expands to nothing --- there the functions aren't annotated in +anyway and moreover the default preprocessor behavior would make this extremely +tricky to implement. + +@attention Due to the way the attributes are combined on Clang < 8 and GCC, in + certain cases the macro may silently accept even arguments that don't have + a corresponding `CORRADE_ENABLE_*` macro defined. To prevent portability + issues, pay extra attention to have a matching @cpp #ifdef @ce guard. +*/ +#if !defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_CLANG_CL) +#define CORRADE_ENABLE(...) _CORRADE_HELPER_PICK(__VA_ARGS__, _CORRADE_ENABLE8, _CORRADE_ENABLE7, _CORRADE_ENABLE6, _CORRADE_ENABLE5, _CORRADE_ENABLE4, _CORRADE_ENABLE3, _CORRADE_ENABLE2, _CORRADE_ENABLE1, )(__VA_ARGS__) +#else +#define CORRADE_ENABLE(...) +#endif + +/* x86 CPUID implementation on GCC/Clang/MSVC. Has to be inlined in the header + because otherwise in the IFUNC scenario it may result in a cross-SO call + that's unsupported on Clang and older GCC, causing a crash in the dynamic + loader during early startup because it calls into a place that's not there + yet. + + Because casually including leads to 37+ kLOC (!!!), I go an + extra way and use inline assembly instead. Which, compared to using the + intrinsics --- funnily enough --- reduces the amount of cursing and lengthy + comments explaining compiler bugs and differences to an absolute minimum. + If any of the following misbehaves, please check Git history for the + original implementation. */ +#if defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC)) +namespace Implementation { + inline void cpuid(int data[4], int leaf, int count) { + /* What's in GCC's / Clang's cpuid.h. Clang-cl as well, as it doesn't + seem to know the __cpuidex() intrinsics. */ + #if defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) + #ifdef CORRADE_TARGET_32BIT + asm("cpuid": \ + "=a"(data[0]), "=b"(data[1]), "=c"(data[2]), "=d"(data[3]): \ + "0"(leaf), "2"(count)); + #else + /* Clang says "x86-64 uses %rbx as the base register", GCC says "%rbx + may be the PIC register", so probably important to preserve it or + some such? ¯\_(ツ)_/¯ */ + asm("xchgq %%rbx,%q1\n" \ + "cpuid\n" \ + "xchgq %%rbx,%q1": \ + "=a"(data[0]), "=b"(data[1]), "=c"(data[2]), "=d"(data[3]): \ + "0"(leaf), "2"(count)); + #endif + + /* Declared at the top of the file */ + #elif defined(CORRADE_TARGET_MSVC) + __cpuidex(data, leaf, count); + #else + #error + #endif + } +} + +inline Features runtimeFeatures() { + union { + struct { + unsigned int ax, bx, cx, dx; + } e; + int data[4]; + } cpuid{}; + + Implementation::cpuid(cpuid.data, 1, 0); + + /* https://en.wikipedia.org/wiki/CPUID#EAX=1:_Processor_Info_and_Feature_Bits */ + unsigned int out = 0; + if(cpuid.e.dx & (1 << 26)) out |= TypeTraits::Index; + if(cpuid.e.cx & (1 << 0)) out |= TypeTraits::Index; + if(cpuid.e.cx & (1 << 9)) out |= TypeTraits::Index; + if(cpuid.e.cx & (1 << 19)) out |= TypeTraits::Index; + if(cpuid.e.cx & (1 << 20)) out |= TypeTraits::Index; + + /* https://en.wikipedia.org/wiki/CPUID#EAX=80000001h:_Extended_Processor_Info_and_Feature_Bits, + bit 5 says "ABM (lzcnt and popcnt)", but + https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation) + says that while LZCNT is advertised in the ABM CPUID bit, POPCNT is a + separate CPUID flag. Get POPCNT first, ABM later. */ + if(cpuid.e.cx & (1 << 23)) out |= TypeTraits::Index; + + /* AVX needs OS support checked, as the OS needs to be capable of saving + and restoring the expanded registers when switching contexts: + https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#Operating_system_support */ + if((cpuid.e.cx & (1 << 27)) && /* XSAVE/XRESTORE CPU support */ + (cpuid.e.cx & (1 << 28))) /* AVX CPU support */ + { + /* XGETBV indicates that the registers will be properly saved and + restored by the OS: https://stackoverflow.com/a/22521619. */ + + /* https://github.com/vectorclass/version2/blob/ff7450acfad9d3a7c6825d92cfb782a42ccfa71f/instrset_detect.cpp#L30-L32 + Clang-cl as well, as it doesn't seem to know the MSVC intrinsics. */ + #if defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) + unsigned int a, d; + __asm("xgetbv": "=a"(a), "=d"(d): "c"(0): ); + const unsigned long long xgetbv = a|(static_cast(d) << 32); + + /* Declared at the top of the file */ + #elif defined(CORRADE_TARGET_MSVC) + const unsigned long long xgetbv = _xgetbv(0); + #else + #error + #endif + + if((xgetbv & 0x06) == 0x6) + out |= TypeTraits::Index; + } + + /* If AVX is not supported, we don't check any following flags either */ + if(out & TypeTraits::Index) { + if(cpuid.e.cx & (1 << 29)) out |= TypeTraits::Index; + if(cpuid.e.cx & (1 << 12)) out |= TypeTraits::Index; + + /* https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features */ + Implementation::cpuid(cpuid.data, 7, 0); + if(cpuid.e.bx & (1 << 3)) out |= TypeTraits::Index; + if(cpuid.e.bx & (1 << 5)) out |= TypeTraits::Index; + if(cpuid.e.bx & (1 << 16)) out |= TypeTraits::Index; + } + + /* And now the LZCNT bit, finally + https://en.wikipedia.org/wiki/CPUID#EAX=80000001h:_Extended_Processor_Info_and_Feature_Bits */ + Implementation::cpuid(cpuid.data, 0x80000001, 0); + if(cpuid.e.cx & (1 << 5)) out |= TypeTraits::Index; + + return Features{out}; +} +#endif + +/* ARM implementation on Linux and Android, inlined for the same reason as the + x86 variant above -- to make IFUNC work. As getauxval() can't be called from + within an ifunc resolver because there it's too early for an external call, + the value of AT_HWCAP is instead passed to it from outside, on glibc 2.13+ + and on Android API 30+: + https://github.com/bminor/glibc/commit/7520ff8c744a704ca39741c165a2360d63a4f47a + https://android.googlesource.com/platform/bionic/+/e949195f6489653ee3771535951ed06973246c3e/libc/include/sys/ifunc.h + Which means we need a variant of runtimeFeatures() that is able to operate + with a value fed from outside, which is then used inside such resolvers. A + nice consequence of that is that we don't need any other headers. + + The public Cpu::runtimeFeatures() is deinlined, calls getauxval() and passes + it into this function. */ +/** @todo If AT_HWCAP2 or other bits are needed, it's passed to ifunc resolvers + only since glibc 2.30 (and Android API 30+, which is the same as before): + https://github.com/bminor/glibc/commit/2b8a3c86e7606cf1b0a997dad8af2d45ae8989c3 */ +#if defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) +namespace Implementation { + inline Features runtimeFeatures(const unsigned long caps) { + unsigned int out = 0; + #ifdef CORRADE_TARGET_32BIT + if(caps & (1 << 12) /*HWCAP_NEON*/) out |= TypeTraits::Index; + /* Since FMA is enabled by passing -mfpu=neon-vfpv4, I assume this is + the flag that corresponds to it. */ + if(caps & (1 << 16) /*HWCAP_VFPv4*/) out |= TypeTraits::Index; + #else + /* On ARM64 NEON and NEON FMA is implicit. For extra security make use + of the CORRADE_TARGET_ defines (which should be always there). */ + out |= + #ifdef CORRADE_TARGET_NEON + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_NEON_FMA + TypeTraits::Index| + #endif + 0; + /* The HWCAP flags are extremely cryptic. The only vague confirmation + is in a *commit message* to the kernel hwcaps file, FFS. The + HWCAP_FPHP seems to correspond to scalar FP16, so the other should + be the vector one? + https://github.com/torvalds/linux/blame/master/arch/arm64/include/uapi/asm/hwcap.h + This one also isn't present on 32-bit, so I assume it's + ARM64-only? */ + if(caps & (1 << 10) /*HWCAP_ASIMDHP*/) out |= TypeTraits::Index; + #endif + return Features{out}; + } +} +#endif + +} + +} + +#endif diff --git a/src/Corrade/PluginManager/CMakeLists.txt b/src/Corrade/PluginManager/CMakeLists.txt index 60b510ea8..6804fead8 100644 --- a/src/Corrade/PluginManager/CMakeLists.txt +++ b/src/Corrade/PluginManager/CMakeLists.txt @@ -74,7 +74,7 @@ elseif(CORRADE_BUILD_STATIC_PIC) set_target_properties(CorradePluginManager PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() # Utility also does -ldl now -target_link_libraries(CorradePluginManager CorradeUtility) +target_link_libraries(CorradePluginManager PUBLIC CorradeUtility) install(TARGETS CorradePluginManager RUNTIME DESTINATION ${CORRADE_BINARY_INSTALL_DIR} @@ -93,9 +93,9 @@ if(CORRADE_BUILD_TESTS) if(CORRADE_BUILD_STATIC_PIC) set_target_properties(CorradePluginManagerTestLib PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() - target_link_libraries(CorradePluginManagerTestLib CorradeUtility) + target_link_libraries(CorradePluginManagerTestLib PUBLIC CorradeUtility) if(CORRADE_TARGET_UNIX) - target_link_libraries(CorradePluginManagerTestLib ${CMAKE_DL_LIBS}) + target_link_libraries(CorradePluginManagerTestLib PUBLIC ${CMAKE_DL_LIBS}) endif() add_subdirectory(Test) diff --git a/src/Corrade/Test/CMakeLists.txt b/src/Corrade/Test/CMakeLists.txt index 733495046..4df49c845 100644 --- a/src/Corrade/Test/CMakeLists.txt +++ b/src/Corrade/Test/CMakeLists.txt @@ -33,6 +33,17 @@ corrade_add_test(MainTest MainTest.cpp # Prefixed with project name to avoid conflicts with TagsTest in Magnum corrade_add_test(CorradeTagsTest TagsTest.cpp) +# Platforms like Emscripten or Android that don't have dynamic plugin support +# don't really have dynamic libraries either. Thus there's no point in trying +# to benchmark a call into an external dynamic library. +if(CORRADE_PLUGINMANAGER_NO_DYNAMIC_PLUGIN_SUPPORT) + add_library(CpuTestExternalLibrary STATIC CpuTestExternalLibrary.cpp) +else() + add_library(CpuTestExternalLibrary SHARED CpuTestExternalLibrary.cpp) +endif() +target_link_libraries(CpuTestExternalLibrary PRIVATE CorradeUtility) +corrade_add_test(CpuTest CpuTest.cpp LIBRARIES CpuTestExternalLibrary) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/configure.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/configure.h) corrade_add_test(TargetTest TargetTest.cpp) diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp new file mode 100644 index 000000000..bcb314441 --- /dev/null +++ b/src/Corrade/Test/CpuTest.cpp @@ -0,0 +1,1922 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include + +#include "Corrade/Cpu.h" +#include "Corrade/Containers/StringView.h" +#include "Corrade/TestSuite/Tester.h" +#include "Corrade/Utility/DebugStl.h" /** @todo remove when is gone */ + +#ifdef CORRADE_ENABLE_SSE2 +#include "Corrade/Utility/IntrinsicsSse2.h" +#endif +#ifdef CORRADE_ENABLE_SSE3 +#include "Corrade/Utility/IntrinsicsSse3.h" +#endif +#ifdef CORRADE_ENABLE_SSSE3 +#include "Corrade/Utility/IntrinsicsSsse3.h" +#endif +#if defined(CORRADE_ENABLE_SSE41) || defined(CORRADE_ENABLE_SSE42) +#include "Corrade/Utility/IntrinsicsSse4.h" +#endif +#if defined(CORRADE_ENABLE_AVX) || defined(CORRADE_ENABLE_AVX_F16C) || defined(CORRADE_ENABLE_AVX_FMA) || defined(CORRADE_ENABLE_AVX2) || defined(CORRADE_ENABLE_AVX512F) +#include "Corrade/Utility/IntrinsicsAvx.h" +#endif +#ifdef CORRADE_ENABLE_NEON +#include +#endif +#ifdef CORRADE_ENABLE_SIMD128 +#include +#endif + +#include "CpuTestExternalLibrary.h" + +namespace Corrade { namespace Test { namespace { + +struct CpuTest: TestSuite::Tester { + explicit CpuTest(); + + void tagNoDefaultConstructor(); + void tagInlineDefinition(); + void tagConstructTemplate(); + + void typeTraits(); + + /* Most of the operator tests is inherited from EnumSetTest, just replacing + Feature::Fast with Cpu::Sse2, Feature::Cheap with Cpu::Sse3, + Feature::Tested with Cpu::Ssse3 and Feature::Popular with Cpu::Sse41, + and adjusting numeric values for inverse because there's no fullValue + specified */ + + void featuresConstructScalar(); + void featuresConstruct(); + void featuresConstructTemplate(); + void featuresOperatorOr(); + void featuresOperatorAnd(); + void featuresOperatorXor(); + void featuresOperatorBoolScalar(); + void featuresOperatorBool(); + void featuresOperatorInverse(); + void featuresCompare(); + + void detectDefault(); + void detect(); + + void bitIndex(); + void bitCount(); + void priority(); + + void tagDispatch(); + void tagDispatchExtraExact(); + void tagDispatchExtraUnusedExtra(); + void tagDispatchExtraFallbackExtra(); + void tagDispatchExtraFallbackBase(); + void tagDispatchExtraFallbackBoth(); + void tagDispatchExtraPriority(); + + void tagDispatchRuntime(); + void tagDispatchRuntimeExtra(); + void tagDispatchRuntimeExtraCombination(); + void tagDispatchRuntimeExtraZeroExtra(); + + void tagDispatchedCompileTime(); + void tagDispatchedPointer(); + void tagDispatchedIfunc(); + + void benchmarkTagDispatchedCompileTime(); + void benchmarkTagDispatchedPointer(); + void benchmarkTagDispatchedIfunc(); + void benchmarkTagDispatchedExternalLibraryCompileTime(); + void benchmarkTagDispatchedExternalLibraryPointer(); + void benchmarkTagDispatchedExternalLibraryIfunc(); + void benchmarkTagDispatchedExternalLibraryEveryCall(); + + template void enableMacros(); + + void enableMacrosMultiple(); + void enableMacrosMultipleAllEmpty(); + + void enableMacrosLambda(); + void enableMacrosLambdaMultiple(); + + void debug(); + void debugPacked(); +}; + +const struct { + const char* name; + Cpu::Features(*function)(); +} DetectData[]{ + {"compiled", Cpu::compiledFeatures}, + {"runtime", Cpu::runtimeFeatures} +}; + +CpuTest::CpuTest() { + addTests({&CpuTest::tagNoDefaultConstructor, + &CpuTest::tagInlineDefinition, + &CpuTest::tagConstructTemplate, + + &CpuTest::typeTraits, + + &CpuTest::featuresConstructScalar, + &CpuTest::featuresConstruct, + &CpuTest::featuresConstructTemplate, + &CpuTest::featuresOperatorOr, + &CpuTest::featuresOperatorAnd, + &CpuTest::featuresOperatorXor, + &CpuTest::featuresOperatorBoolScalar, + &CpuTest::featuresOperatorBool, + &CpuTest::featuresOperatorInverse, + &CpuTest::featuresCompare, + + &CpuTest::detectDefault}); + + addInstancedTests({&CpuTest::detect}, + Containers::arraySize(DetectData)); + + addTests({&CpuTest::bitIndex, + &CpuTest::bitCount, + &CpuTest::priority, + + &CpuTest::tagDispatch, + &CpuTest::tagDispatchExtraExact, + &CpuTest::tagDispatchExtraUnusedExtra, + &CpuTest::tagDispatchExtraFallbackBase, + &CpuTest::tagDispatchExtraFallbackExtra, + &CpuTest::tagDispatchExtraFallbackBoth, + &CpuTest::tagDispatchExtraPriority, + + &CpuTest::tagDispatchRuntime, + &CpuTest::tagDispatchRuntimeExtra, + &CpuTest::tagDispatchRuntimeExtraCombination, + &CpuTest::tagDispatchRuntimeExtraZeroExtra, + + &CpuTest::tagDispatchedCompileTime, + &CpuTest::tagDispatchedPointer, + &CpuTest::tagDispatchedIfunc}); + + addBenchmarks({&CpuTest::benchmarkTagDispatchedCompileTime, + &CpuTest::benchmarkTagDispatchedPointer, + &CpuTest::benchmarkTagDispatchedIfunc, + &CpuTest::benchmarkTagDispatchedExternalLibraryCompileTime, + &CpuTest::benchmarkTagDispatchedExternalLibraryPointer, + &CpuTest::benchmarkTagDispatchedExternalLibraryIfunc, + &CpuTest::benchmarkTagDispatchedExternalLibraryEveryCall}, 100); + + addTests({ + #ifdef CORRADE_TARGET_X86 + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + #elif defined(CORRADE_TARGET_ARM) + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + #elif defined(CORRADE_TARGET_WASM) + &CpuTest::enableMacros, + #endif + + &CpuTest::enableMacrosMultiple, + &CpuTest::enableMacrosMultipleAllEmpty, + + &CpuTest::enableMacrosLambda, + &CpuTest::enableMacrosLambdaMultiple, + + &CpuTest::debug, + &CpuTest::debugPacked}); +} + +using namespace Containers::Literals; + +void CpuTest::tagNoDefaultConstructor() { + /* Isn't default constructible to prevent ambiguity when calling + foo({}) if both foo(TagT) and foo(whatever) is available */ + CORRADE_VERIFY(!std::is_default_constructible::value); + #ifdef CORRADE_TARGET_X86 + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + #elif defined(CORRADE_TARGET_ARM) + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + #elif defined(CORRADE_TARGET_WASM) + CORRADE_VERIFY(!std::is_default_constructible::value); + #endif +} + +void CpuTest::tagInlineDefinition() { + /* Just a sanity check that the types match */ + CORRADE_VERIFY(std::is_same::value); + #ifdef CORRADE_TARGET_X86 + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + #elif defined(CORRADE_TARGET_ARM) + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + #elif defined(CORRADE_TARGET_WASM) + CORRADE_VERIFY(std::is_same::value); + #endif +} + +void CpuTest::tagConstructTemplate() { + #ifdef CORRADE_TARGET_X86 + auto tag = Cpu::tag(); + constexpr auto cTag = Cpu::tag(); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + #elif defined(CORRADE_TARGET_ARM) + auto tag = Cpu::tag(); + constexpr auto cTag = Cpu::tag(); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + #elif defined(CORRADE_TARGET_WASM) + auto tag = Cpu::tag(); + constexpr auto cTag = Cpu::tag(); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + #else + CORRADE_SKIP("No Cpu tags available on this platform"); + #endif +} + +void CpuTest::typeTraits() { + CORRADE_VERIFY(!Cpu::TypeTraits::Index); + #ifdef CORRADE_TARGET_X86 + CORRADE_VERIFY(Cpu::TypeTraits::Index); + CORRADE_COMPARE(Cpu::TypeTraits::name(), "Avx2"_s); + #elif defined(CORRADE_TARGET_ARM) + CORRADE_VERIFY(Cpu::TypeTraits::Index); + CORRADE_COMPARE(Cpu::TypeTraits::name(), "NeonFp16"_s); + #elif defined(CORRADE_TARGET_WASM) + CORRADE_VERIFY(Cpu::TypeTraits::Index); + CORRADE_COMPARE(Cpu::TypeTraits::name(), "Simd128"_s); + #else + CORRADE_SKIP("No Cpu tags available on this platform"); + #endif +} + +void CpuTest::featuresConstructScalar() { + Cpu::Features noFeatures1; + Cpu::Features noFeatures2 = Cpu::Scalar; + constexpr Cpu::Features cNoFeatures1; + constexpr Cpu::Features cNoFeatures2 = Cpu::Scalar; + CORRADE_COMPARE(std::uint32_t(noFeatures1), 0); + CORRADE_COMPARE(std::uint32_t(noFeatures2), 0); + CORRADE_COMPARE(std::uint32_t(cNoFeatures1), 0); + CORRADE_COMPARE(std::uint32_t(cNoFeatures2), 0); + + CORRADE_VERIFY(std::is_nothrow_constructible::value); + CORRADE_VERIFY(std::is_nothrow_constructible::value); +} + +void CpuTest::featuresConstruct() { + #ifdef CORRADE_TARGET_X86 + Cpu::Features features = Cpu::Sse3; + constexpr Cpu::Features cFeatures = Cpu::Sse3; + CORRADE_COMPARE(std::uint32_t(features), 2); + CORRADE_COMPARE(std::uint32_t(cFeatures), 2); + CORRADE_VERIFY(std::is_nothrow_constructible::value); + #elif defined(CORRADE_TARGET_ARM) + Cpu::Features features = Cpu::Neon; + constexpr Cpu::Features cFeatures = Cpu::Neon; + CORRADE_COMPARE(std::uint32_t(features), 1); + CORRADE_COMPARE(std::uint32_t(cFeatures), 1); + CORRADE_VERIFY(std::is_nothrow_constructible::value); + #elif defined(CORRADE_TARGET_WASM) + Cpu::Features features = Cpu::Simd128; + constexpr Cpu::Features cFeatures = Cpu::Simd128; + CORRADE_COMPARE(std::uint32_t(features), 1); + CORRADE_COMPARE(std::uint32_t(cFeatures), 1); + CORRADE_VERIFY(std::is_nothrow_constructible::value); + #else + CORRADE_SKIP("No Cpu tags available on this platform"); + #endif +} + +void CpuTest::featuresConstructTemplate() { + #ifdef CORRADE_TARGET_X86 + auto features = Cpu::features(); + constexpr auto cFeatures = Cpu::features(); + CORRADE_COMPARE(std::uint32_t(features), 2); + CORRADE_COMPARE(std::uint32_t(cFeatures), 2); + #elif defined(CORRADE_TARGET_ARM) + auto features = Cpu::features(); + constexpr auto cFeatures = Cpu::features(); + CORRADE_COMPARE(std::uint32_t(features), 1); + CORRADE_COMPARE(std::uint32_t(cFeatures), 1); + #elif defined(CORRADE_TARGET_WASM) + auto features = Cpu::features(); + constexpr auto cFeatures = Cpu::features(); + CORRADE_COMPARE(std::uint32_t(features), 1); + CORRADE_COMPARE(std::uint32_t(cFeatures), 1); + #else + CORRADE_SKIP("No Cpu tags available on this platform"); + #endif +} + +void CpuTest::featuresOperatorOr() { + #ifdef CORRADE_TARGET_X86 + /* This is actually using the compile-time operation, producing Tags, + which is tested explicitly below, but that should be completely + transparent to the user and work as if it produced Features directly + instead of going through Tags first */ + Cpu::Features features = Cpu::Sse3|Cpu::Sse2; + CORRADE_COMPARE(std::uint32_t(features), 3); + + CORRADE_COMPARE(std::uint32_t(features|Cpu::Ssse3), 7); + CORRADE_COMPARE(std::uint32_t(Cpu::Ssse3|features), 7); + + features |= Cpu::Ssse3; + CORRADE_COMPARE(std::uint32_t(features), 7); + + constexpr Cpu::Features cFeatures = Cpu::Sse3|Cpu::Sse2; + constexpr Cpu::Features cFeatures1 = cFeatures|Cpu::Ssse3; + constexpr Cpu::Features cFeatures2 = Cpu::Ssse3|cFeatures; + CORRADE_COMPARE(std::uint32_t(cFeatures), 3); + CORRADE_COMPARE(std::uint32_t(cFeatures1), 7); + CORRADE_COMPARE(std::uint32_t(cFeatures2), 7); + + /* Test also the compile-time operation, different values should be + different types but same values should be same types */ + constexpr auto cTags = Cpu::Sse3|Cpu::Sse2; + constexpr auto cTags1 = cTags|Cpu::Ssse3; + constexpr auto cTags2 = Cpu::Ssse3|cTags; + CORRADE_COMPARE(std::uint32_t(cTags), 3); + CORRADE_COMPARE(std::uint32_t(cTags1), 7); + CORRADE_COMPARE(std::uint32_t(cTags2), 7); + CORRADE_VERIFY(!std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + + /* And also with Tags<> on both sides, check that it doesn't decay to an + int or other horrible thing */ + constexpr auto cTags3 = Cpu::Ssse3|Cpu::Sse41; + constexpr auto cTags4 = cTags3|cTags; + CORRADE_COMPARE(std::uint32_t(cTags4), 15); + CORRADE_VERIFY(std::is_same>::value); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +void CpuTest::featuresOperatorAnd() { + #ifdef CORRADE_TARGET_X86 + /* This is actually using the compile-time operation, producing Tags, + which is tested explicitly below, but that should be completely + transparent to the user and work as if it produced Features directly + instead of going through Tags first */ + CORRADE_COMPARE(std::uint32_t(Cpu::Sse3 & Cpu::Sse2), 0); + + Cpu::Features features = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + CORRADE_COMPARE(std::uint32_t(features & Cpu::Sse41), 8); + CORRADE_COMPARE(std::uint32_t(Cpu::Sse41 & features), 8); + + CORRADE_COMPARE(std::uint32_t(features & Cpu::Ssse3), 0); + + Cpu::Features features2 = Cpu::Sse41|Cpu::Sse2|Cpu::Ssse3; + CORRADE_COMPARE(std::uint32_t(features & features2), 9); + + features &= features2; + CORRADE_COMPARE(std::uint32_t(features), 9); + + constexpr Cpu::Features cFeatures = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + constexpr Cpu::Features cFeatures1 = cFeatures & Cpu::Sse41; + constexpr Cpu::Features cFeatures2 = Cpu::Sse41 & cFeatures; + CORRADE_COMPARE(std::uint32_t(cFeatures1), 8); + CORRADE_COMPARE(std::uint32_t(cFeatures2), 8); + + /* Test also the compile-time operation, different values should be + different types but same values should be same types */ + constexpr auto cTags = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + constexpr auto cTags1 = cTags & Cpu::Sse41; + constexpr auto cTags2 = Cpu::Sse41 & cTags; + CORRADE_COMPARE(std::uint32_t(cTags1), 8); + CORRADE_COMPARE(std::uint32_t(cTags2), 8); + CORRADE_VERIFY(!std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + + /* And also with Tags<> on both sides, check that it doesn't decay to an + int or other horrible thing */ + constexpr auto cTags3 = Cpu::Ssse3|Cpu::Sse41; + constexpr auto cTags4 = cTags3 & cTags; + CORRADE_COMPARE(std::uint32_t(cTags4), 8); + CORRADE_VERIFY(std::is_same>::value); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +void CpuTest::featuresOperatorXor() { + #ifdef CORRADE_TARGET_X86 + /* This is actually using the compile-time operation, producing Tags, + which is tested explicitly below, but that should be completely + transparent to the user and work as if it produced Features directly + instead of going through Tags first */ + CORRADE_COMPARE(std::uint32_t(Cpu::Sse3 ^ Cpu::Sse3), 0); + CORRADE_COMPARE(std::uint32_t(Cpu::Sse3 ^ Cpu::Sse2), 3); + + Cpu::Features features = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + CORRADE_COMPARE(std::uint32_t(features ^ Cpu::Sse2), 10); + CORRADE_COMPARE(std::uint32_t(Cpu::Sse2 ^ features), 10); + + CORRADE_COMPARE(std::uint32_t(features ^ Cpu::Sse41), 3); + + Cpu::Features features2 = Cpu::Sse41|Cpu::Sse2|Cpu::Ssse3; + CORRADE_COMPARE(std::uint32_t(features ^ features2), 6); + + features ^= features2; + CORRADE_COMPARE(std::uint32_t(features), 6); + + constexpr Cpu::Features cFeatures = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + constexpr Cpu::Features cFeatures1 = cFeatures ^ Cpu::Sse2; + constexpr Cpu::Features cFeatures2 = Cpu::Sse2 ^ cFeatures; + CORRADE_COMPARE(std::uint32_t(cFeatures1), 10); + CORRADE_COMPARE(std::uint32_t(cFeatures2), 10); + + /* Test also the compile-time operation, different values should be + different types but same values should be same types */ + constexpr auto cTags = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + constexpr auto cTags1 = cTags ^ Cpu::Sse2; + constexpr auto cTags2 = Cpu::Sse2 ^ cTags; + CORRADE_COMPARE(std::uint32_t(cTags1), 10); + CORRADE_COMPARE(std::uint32_t(cTags2), 10); + CORRADE_VERIFY(!std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + + /* And also with Tags<> on both sides, check that it doesn't decay to an + int or other horrible thing */ + constexpr auto cTags3 = Cpu::Ssse3|Cpu::Sse41; + constexpr auto cTags4 = cTags3 ^ cTags; + CORRADE_COMPARE(std::uint32_t(cTags4), 7); + CORRADE_VERIFY(std::is_same>::value); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +void CpuTest::featuresOperatorBoolScalar() { + CORRADE_COMPARE(!!Cpu::Features{Cpu::Scalar}, false); + + constexpr bool cFeatures = !!Cpu::Features{Cpu::Scalar}; + CORRADE_VERIFY(!cFeatures); + + /* Have to use an implementation detail to create the Tags type here. Which + is fine, people shouldn't need to do this directly. */ + constexpr bool cTags = !!Cpu::Implementation::tags(Cpu::Scalar); + CORRADE_VERIFY(!cTags); +} + +void CpuTest::featuresOperatorBool() { + #ifdef CORRADE_TARGET_X86 + Cpu::Features features = Cpu::Sse3|Cpu::Sse2; + CORRADE_COMPARE(!!(features & Cpu::Sse41), false); + CORRADE_COMPARE(!!(features & Cpu::Sse3), true); + + constexpr Cpu::Features cFeatures = Cpu::Sse3|Cpu::Sse2; + constexpr bool cFeatures1 = !!(cFeatures & Cpu::Sse41); + constexpr bool cFeatures2 = !!(cFeatures & Cpu::Sse3); + CORRADE_VERIFY(!cFeatures1); + CORRADE_VERIFY(cFeatures2); + + /* Test also the compile-time operation */ + constexpr auto cTags = Cpu::Sse3|Cpu::Sse2; + constexpr bool cTags1 = !!(cTags & Cpu::Sse41); + constexpr bool cTags2 = !!(cTags & Cpu::Sse3); + CORRADE_VERIFY(!cTags1); + CORRADE_VERIFY(cTags2); + CORRADE_VERIFY(!std::is_same::value); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +void CpuTest::featuresOperatorInverse() { + #ifdef CORRADE_TARGET_X86 + /* This is actually using the compile-time operation, producing Tags, + which is tested explicitly below, but that should be completely + transparent to the user and work as if it produced Features directly + instead of going through Tags first */ + CORRADE_COMPARE(std::uint32_t(~Cpu::Scalar), 0xffffffffu); + CORRADE_COMPARE(std::uint32_t(~(Cpu::Sse41|Cpu::Sse3)), 4294967285u); + CORRADE_COMPARE(std::uint32_t(~Cpu::Sse41), 4294967287u); + + constexpr Cpu::Features cFeatures1 = ~Cpu::Scalar; + constexpr Cpu::Features cFeatures2 = ~(Cpu::Sse41|Cpu::Sse3); + CORRADE_COMPARE(std::uint32_t(cFeatures1), 0xffffffffu); + CORRADE_COMPARE(std::uint32_t(cFeatures2), 4294967285u); + + /* Test also the compile-time operation, different values should be + different types but same values should be same types */ + constexpr auto cTags1 = ~Cpu::Scalar; + constexpr auto cTags2 = ~(Cpu::Sse41|Cpu::Sse3); + CORRADE_COMPARE(std::uint32_t(cTags1), 0xffffffffu); + CORRADE_COMPARE(std::uint32_t(cTags2), 4294967285u); + CORRADE_VERIFY(!std::is_same::value); + CORRADE_VERIFY(!std::is_same::value); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +void CpuTest::featuresCompare() { + #ifdef CORRADE_TARGET_X86 + Cpu::Features features = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + CORRADE_VERIFY(features == features); + CORRADE_VERIFY(!(features != features)); + CORRADE_VERIFY(Cpu::Sse3 == Cpu::Sse3); + CORRADE_VERIFY(Cpu::Sse3 != Cpu::Sse41); + + CORRADE_VERIFY(Cpu::Scalar <= Cpu::Sse41); + CORRADE_VERIFY(Cpu::Sse41 >= Cpu::Scalar); + CORRADE_VERIFY(Cpu::Sse41 <= Cpu::Sse41); + CORRADE_VERIFY(Cpu::Sse41 >= Cpu::Sse41); + CORRADE_VERIFY(Cpu::Sse41 <= features); + CORRADE_VERIFY(features >= Cpu::Sse41); + CORRADE_VERIFY(features <= features); + CORRADE_VERIFY(features >= features); + + CORRADE_VERIFY(features <= (Cpu::Sse41|Cpu::Sse2|Cpu::Sse3|Cpu::Ssse3)); + CORRADE_VERIFY(!(features >= (Cpu::Sse41|Cpu::Sse2|Cpu::Sse3|Cpu::Ssse3))); + + constexpr Cpu::Features cFeatures = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + constexpr bool cFeaturesEqual = cFeatures == cFeatures; + constexpr bool cFeaturesNonEqual = cFeatures != cFeatures; + constexpr bool cFeaturesLessEqual = cFeatures <= cFeatures; + constexpr bool cFeaturesGreaterEqual = cFeatures >= cFeatures; + CORRADE_VERIFY(cFeaturesEqual); + CORRADE_VERIFY(!cFeaturesNonEqual); + CORRADE_VERIFY(cFeaturesLessEqual); + CORRADE_VERIFY(cFeaturesGreaterEqual); + + constexpr auto cTags = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + constexpr bool cTagsEqual = cTags == cTags; + constexpr bool cTagsNonEqual = cTags != cTags; + constexpr bool cTagsLessEqual = cTags <= cTags; + constexpr bool cTagsGreaterEqual = cTags >= cTags; + CORRADE_VERIFY(cTagsEqual); + CORRADE_VERIFY(!cTagsNonEqual); + CORRADE_VERIFY(cTagsLessEqual); + CORRADE_VERIFY(cTagsGreaterEqual); + CORRADE_VERIFY(!std::is_same::value); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +void CpuTest::detectDefault() { + CORRADE_INFO("Detected:" << Debug::packed << Cpu::DefaultBase); + + /* There should be at least something if we have any of the defines + present */ + #if defined(CORRADE_TARGET_SSE2) || defined(CORRADE_TARGET_SSE3) || defined(CORRADE_TARGET_SSSE3) || defined(CORRADE_TARGET_SSE41) || defined(CORRADE_TARGET_SSE42) || defined(CORRADE_TARGET_AVX) || defined(CORRADE_TARGET_AVX2) || defined(CORRADE_TARGET_NEON) || defined(CORRADE_TARGET_SIMD128) + CORRADE_VERIFY(Cpu::Features{Cpu::DefaultBase}); + + /* And nothing if we don't */ + #else + CORRADE_VERIFY(!Cpu::Features{Cpu::DefaultBase}); + #endif +} + +void CpuTest::detect() { + auto&& data = DetectData[testCaseInstanceId()]; + setTestCaseDescription(data.name); + + Cpu::Features features = data.function(); + CORRADE_INFO("Detected:" << Debug::packed << features); + + /* The compile-time feature should be listed among these as well, otherwise + we wouldn't even be able to run the code. */ + CORRADE_VERIFY(features >= Cpu::DefaultBase); + + #ifdef CORRADE_TARGET_X86 + /* Test that for every feature, the subset is present as well */ + if(features & Cpu::Avx512f) CORRADE_VERIFY(features & Cpu::Avx2); + if(features & Cpu::Avx2) CORRADE_VERIFY(features & Cpu::Avx); + if(features & Cpu::Avx) CORRADE_VERIFY(features & Cpu::Sse42); + if(features & Cpu::Sse42) CORRADE_VERIFY(features & Cpu::Sse41); + if(features & Cpu::Sse41) CORRADE_VERIFY(features & Cpu::Ssse3); + if(features & Cpu::Ssse3) CORRADE_VERIFY(features & Cpu::Sse3); + if(features & Cpu::Sse3) CORRADE_VERIFY(features & Cpu::Sse2); + #elif defined(CORRADE_TARGET_ARM) + if(features & Cpu::NeonFp16) CORRADE_VERIFY(features & Cpu::NeonFma); + if(features & Cpu::NeonFma) CORRADE_VERIFY(features & Cpu::Neon); + #else + /* WebAssembly currently has just one feature, so no subset testing applies + on those */ + #endif +} + +void CpuTest::bitIndex() { + CORRADE_COMPARE(Cpu::Implementation::BitIndex<0>::Value, 0); + CORRADE_COMPARE(Cpu::Implementation::BitIndex<1>::Value, 1); + CORRADE_COMPARE(Cpu::Implementation::BitIndex<(1 << 7)>::Value, 8); + CORRADE_COMPARE(Cpu::Implementation::BitIndex<(1 << 15)>::Value, 16); +} + +void CpuTest::bitCount() { + CORRADE_COMPARE(Cpu::Implementation::BitCount<0>::Value, 0); + CORRADE_COMPARE(Cpu::Implementation::BitCount<(1 << 7)>::Value, 1); + CORRADE_COMPARE(Cpu::Implementation::BitCount<12345>::Value, 6); + CORRADE_COMPARE(Cpu::Implementation::BitCount<65432>::Value, 11); + CORRADE_COMPARE(Cpu::Implementation::BitCount<0xffffu>::Value, 16); +} + +template unsigned int priorityValue(Cpu::Implementation::Priority) { + return i; +} + +void CpuTest::priority() { + #ifdef CORRADE_TARGET_X86 + /* Extra tag alone is always 1 */ + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::AvxFma)), 1); + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Popcnt)), 1); + + /* More extra tags together is their count */ + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::AvxFma|Cpu::AvxF16c|Cpu::Lzcnt)), 3); + + /* Base tag alone is its BitIndex, where the Scalar is the lowest, thus + zero, times the count of extra tags plus one */ + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Scalar)), 0); + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Sse2)), 1*6); + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Avx2)), 7*6); + + /* Base tag + extra tags is a sum of the two */ + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Avx2|Cpu::AvxFma|Cpu::AvxF16c)), 7*6 + 2); + #elif defined(CORRADE_TARGET_ARM) + /* Base tag alone is its BitIndex, where the Scalar is the lowest, thus + zero, times one as there are no extra tags */ + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Scalar)), 0); + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Neon)), 1); + #elif defined(CORRADE_TARGET_WASM) + /* Base tag alone is its BitIndex, where the Scalar is the lowest, thus + zero, times one as there are no extra tags */ + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Scalar)), 0); + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Simd128)), 1); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#if defined(CORRADE_TARGET_X86) || defined(CORRADE_TARGET_ARM) +const char* dispatch(Cpu::ScalarT) { return "scalar"; } +#ifdef CORRADE_TARGET_X86 +const char* dispatch(Cpu::Sse3T) { return "SSE3"; } +const char* dispatch(Cpu::Avx2T) { return "AVX2"; } +#elif defined(CORRADE_TARGET_ARM) +const char* dispatch(Cpu::NeonT) { return "NEON"; } +const char* dispatch(Cpu::NeonFmaT) { return "NEON FMA"; } +#else +#error +#endif +#endif + +void CpuTest::tagDispatch() { + #ifdef CORRADE_TARGET_X86 + /* If no match, gets the next highest available */ + CORRADE_COMPARE(dispatch(Cpu::Avx512f), "AVX2"_s); + CORRADE_COMPARE(dispatch(Cpu::Sse42), "SSE3"_s); + + /* Exact match */ + CORRADE_COMPARE(dispatch(Cpu::Sse3), "SSE3"_s); + + /* Anything below gets ... the scalar */ + CORRADE_COMPARE(dispatch(Cpu::Sse2), "scalar"_s); + #elif defined(CORRADE_TARGET_ARM) + /* If no match, gets the next highest available */ + CORRADE_COMPARE(dispatch(Cpu::NeonFp16), "NEON FMA"_s); + + /* Exact match */ + CORRADE_COMPARE(dispatch(Cpu::Neon), "NEON"_s); + CORRADE_COMPARE(dispatch(Cpu::Scalar), "scalar"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#ifdef CORRADE_TARGET_X86 +const char* dispatchExtraExact(CORRADE_CPU_DECLARE(Cpu::Avx2|Cpu::AvxFma|Cpu::AvxF16c)) { return "AVX2+FMA+F16C"; } +const char* dispatchExtraExact(CORRADE_CPU_DECLARE(Cpu::Sse42|Cpu::Popcnt)) { return "SSE4.2+POPCNT"; } +const char* dispatchExtraExact(CORRADE_CPU_DECLARE(Cpu::Ssse3)) { return "SSSE3"; } +const char* dispatchExtraExact(CORRADE_CPU_DECLARE(Cpu::Popcnt|Cpu::Lzcnt)) { return "POPCNT+LZCNT"; } +#endif + +void CpuTest::tagDispatchExtraExact() { + #ifdef CORRADE_TARGET_X86 + /* For each there's an exact matching overload */ + CORRADE_COMPARE(dispatchExtraExact(CORRADE_CPU_SELECT(Cpu::Avx2|Cpu::AvxFma|Cpu::AvxF16c)), "AVX2+FMA+F16C"_s); + CORRADE_COMPARE(dispatchExtraExact(CORRADE_CPU_SELECT(Cpu::Sse42|Cpu::Popcnt)), "SSE4.2+POPCNT"_s); + CORRADE_COMPARE(dispatchExtraExact(CORRADE_CPU_SELECT(Cpu::Ssse3)), "SSSE3"_s); + /* The base tag doesn't even need to be there */ + CORRADE_COMPARE(dispatchExtraExact(CORRADE_CPU_SELECT(Cpu::Popcnt|Cpu::Lzcnt)), "POPCNT+LZCNT"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#ifdef CORRADE_TARGET_X86 +const char* dispatchExtraUnusedExtra(CORRADE_CPU_DECLARE(Cpu::Avx2)) { return "AVX2"; } +const char* dispatchExtraUnusedExtra(CORRADE_CPU_DECLARE(Cpu::Sse42)) { return "SSE4.2"; } +const char* dispatchExtraUnusedExtra(CORRADE_CPU_DECLARE(Cpu::Ssse3)) { return "SSSE3"; } +const char* dispatchExtraUnusedExtra(CORRADE_CPU_DECLARE(Cpu::Scalar)) { return "scalar"; } +#endif + +void CpuTest::tagDispatchExtraUnusedExtra() { + #ifdef CORRADE_TARGET_X86 + /* The extra tags get ignored, only the base one will be used */ + CORRADE_COMPARE(dispatchExtraUnusedExtra(CORRADE_CPU_SELECT(Cpu::Avx2|Cpu::AvxFma|Cpu::AvxF16c)), "AVX2"_s); + CORRADE_COMPARE(dispatchExtraUnusedExtra(CORRADE_CPU_SELECT(Cpu::Sse42|Cpu::Popcnt)), "SSE4.2"_s); + CORRADE_COMPARE(dispatchExtraUnusedExtra(CORRADE_CPU_SELECT(Cpu::Ssse3)), "SSSE3"_s); + /* The base tag doesn't even need to be there */ + CORRADE_COMPARE(dispatchExtraUnusedExtra(CORRADE_CPU_SELECT(Cpu::Popcnt|Cpu::Lzcnt)), "scalar"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#ifdef CORRADE_TARGET_X86 +const char* dispatchExtraFallbackExtra(CORRADE_CPU_DECLARE(Cpu::Avx2|Cpu::Popcnt|Cpu::Lzcnt)) { return "AVX2+POPCNT+LZCNT"; } +const char* dispatchExtraFallbackExtra(CORRADE_CPU_DECLARE(Cpu::Sse42|Cpu::Lzcnt)) { return "SSE4.2+LZCNT"; } +const char* dispatchExtraFallbackExtra(CORRADE_CPU_DECLARE(Cpu::Sse42)) { return "SSE4.2"; } +#endif + +void CpuTest::tagDispatchExtraFallbackExtra() { + #ifdef CORRADE_TARGET_X86 + /* The base tag stays the same, but the extra ones get dropped */ + CORRADE_COMPARE(dispatchExtraFallbackExtra(CORRADE_CPU_SELECT(Cpu::Avx2|Cpu::Popcnt|Cpu::Lzcnt|Cpu::AvxFma|Cpu::AvxF16c)), "AVX2+POPCNT+LZCNT"_s); + CORRADE_COMPARE(dispatchExtraFallbackExtra(CORRADE_CPU_SELECT(Cpu::Sse42|Cpu::Popcnt|Cpu::Lzcnt)), "SSE4.2+LZCNT"_s); + CORRADE_COMPARE(dispatchExtraFallbackExtra(CORRADE_CPU_SELECT(Cpu::Sse42|Cpu::Popcnt)), "SSE4.2"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#ifdef CORRADE_TARGET_X86 +const char* dispatchExtraFallbackBase(CORRADE_CPU_DECLARE(Cpu::Avx2|Cpu::Popcnt|Cpu::Lzcnt)) { return "AVX2+POPCNT+LZCNT"; } +const char* dispatchExtraFallbackBase(CORRADE_CPU_DECLARE(Cpu::Sse42|Cpu::Lzcnt)) { return "SSE42+LZCNT"; } +const char* dispatchExtraFallbackBase(CORRADE_CPU_DECLARE(Cpu::Ssse3)) { return "SSSE3"; } +const char* dispatchExtraFallbackBase(CORRADE_CPU_DECLARE(Cpu::Popcnt|Cpu::Lzcnt)) { return "POPCNT+LZCNT"; } +#endif + +void CpuTest::tagDispatchExtraFallbackBase() { + #ifdef CORRADE_TARGET_X86 + /* The extra tags stay the same, but the base one gets lowered */ + CORRADE_COMPARE(dispatchExtraFallbackBase(CORRADE_CPU_SELECT(Cpu::Avx512f|Cpu::Popcnt|Cpu::Lzcnt)), "AVX2+POPCNT+LZCNT"_s); + CORRADE_COMPARE(dispatchExtraFallbackBase(CORRADE_CPU_SELECT(Cpu::Avx2|Cpu::Lzcnt)), "SSE42+LZCNT"_s); + CORRADE_COMPARE(dispatchExtraFallbackBase(CORRADE_CPU_SELECT(Cpu::Sse42)), "SSSE3"_s); + CORRADE_COMPARE(dispatchExtraFallbackBase(CORRADE_CPU_SELECT(Cpu::Sse3|Cpu::Popcnt|Cpu::Lzcnt)), "POPCNT+LZCNT"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#ifdef CORRADE_TARGET_X86 +const char* dispatchExtraFallbackBoth(CORRADE_CPU_DECLARE(Cpu::Avx2|Cpu::AvxFma)) { return "AVX2+FMA"; } +const char* dispatchExtraFallbackBoth(CORRADE_CPU_DECLARE(Cpu::Avx|Cpu::AvxF16c)) { return "AVX+F16C"; } +const char* dispatchExtraFallbackBoth(CORRADE_CPU_DECLARE(Cpu::Avx)) { return "AVX"; } +const char* dispatchExtraFallbackBoth(CORRADE_CPU_DECLARE(Cpu::AvxF16c|Cpu::AvxFma)) { return "F16C+FMA"; } +#endif + +void CpuTest::tagDispatchExtraFallbackBoth() { + #ifdef CORRADE_TARGET_X86 + /* Top-class HW, just pick AVX2 as it's the closest */ + CORRADE_COMPARE(dispatchExtraFallbackBoth(CORRADE_CPU_SELECT(Cpu::Avx512f|Cpu::AvxFma|Cpu::AvxF16c)), "AVX2+FMA"_s); + + /* We have one extra less than required for the AVX2 variant, fall back to + AVX with F16C */ + CORRADE_COMPARE(dispatchExtraFallbackBoth(CORRADE_CPU_SELECT(Cpu::Avx2|Cpu::AvxF16c)), "AVX+F16C"_s); + + /* We have AVX2, but neither of the extra bits, just some irrelevant ones, + take plain AVX */ + CORRADE_COMPARE(dispatchExtraFallbackBoth(CORRADE_CPU_SELECT(Cpu::Avx2|Cpu::Lzcnt|Cpu::Popcnt)), "AVX"_s); + + /* We have only SSE3 but both extra bits, fall back to the scalar version + that has them. Yes, it's silly, but the scalar fallback needs to be + verified. */ + CORRADE_COMPARE(dispatchExtraFallbackBoth(CORRADE_CPU_SELECT(Cpu::Sse3|Cpu::AvxF16c|Cpu::AvxFma)), "F16C+FMA"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#ifdef CORRADE_TARGET_X86 +const char* dispatchExtraPriority(CORRADE_CPU_DECLARE(Cpu::Sse42|Cpu::Popcnt|Cpu::Lzcnt)) { + return "SSE4.2+POPCNT+LZCNT"; +} +const char* dispatchExtraPriority(CORRADE_CPU_DECLARE(Cpu::Sse42|Cpu::Popcnt)) { + return "SSE4.2+POPCNT"; +} +const char* dispatchExtraPriority(CORRADE_CPU_DECLARE(Cpu::Sse42|Cpu::Lzcnt)) { + return "SSE4.2+LZCNT"; +} +#endif + +void CpuTest::tagDispatchExtraPriority() { + #ifdef CORRADE_TARGET_X86 + /* The candidate which has the most tags gets picked. OTOH, if it wouldn't + be there, this call would be ambiguous. */ + CORRADE_COMPARE(dispatchExtraPriority(CORRADE_CPU_SELECT(Cpu::Sse42|Cpu::Popcnt|Cpu::Lzcnt)), "SSE4.2+POPCNT+LZCNT"_s); + + /* Both single-extra-tag candidates have the same calculated priority, the + one for which we actually have the feature gets picked */ + CORRADE_COMPARE(dispatchExtraPriority(CORRADE_CPU_SELECT(Cpu::Sse42|Cpu::Popcnt)), "SSE4.2+POPCNT"_s); + CORRADE_COMPARE(dispatchExtraPriority(CORRADE_CPU_SELECT(Cpu::Sse42|Cpu::Lzcnt)), "SSE4.2+LZCNT"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +/* The lambda wrappers are marked with CORRADE_ALWAYS_INLINE to make them go + away with optimizations */ +#if defined(CORRADE_TARGET_X86) || defined(CORRADE_TARGET_ARM) || defined(CORRADE_TARGET_WASM) +typedef const char*(*DispatchRuntimeT)(); + +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntime(Cpu::ScalarT) { + return []() -> const char* { return "scalar"; }; +} +#ifdef CORRADE_TARGET_X86 +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntime(Cpu::Sse3T) { + return []() -> const char* { return "SSE3"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntime(Cpu::Avx2T) { + return []() -> const char* { return "AVX2"; }; +} +#elif defined(CORRADE_TARGET_ARM) +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntime(Cpu::NeonT) { + return []() -> const char* { return "NEON"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntime(Cpu::NeonFmaT) { + return []() -> const char* { return "NEON FMA"; }; +} +#elif defined(CORRADE_TARGET_WASM) +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntime(Cpu::Simd128T) { + return []() -> const char* { return "SIMD128"; }; +} +#else +#error +#endif + +CORRADE_CPU_DISPATCHER_BASE(dispatchRuntime) +#endif + +void CpuTest::tagDispatchRuntime() { + /* Explicitly casting to Cpu::Features here to ensure it's indeed going + through the runtime switch and not directly. Otherwise it's the same as + tagDispatch(). */ + + #ifdef CORRADE_TARGET_X86 + /* If no match, gets the next highest available */ + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::Avx512f})(), "AVX2"_s); + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::Sse42})(), "SSE3"_s); + + /* Exact match */ + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::Sse3})(), "SSE3"_s); + + /* Anything below gets ... the scalar */ + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::Sse2})(), "scalar"_s); + #elif defined(CORRADE_TARGET_ARM) + /* If no match, gets the next highest available */ + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::NeonFp16})(), "NEON FMA"_s); + + /* Exact match */ + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::Neon})(), "NEON"_s); + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::Scalar})(), "scalar"_s); + #elif defined(CORRADE_TARGET_WASM) + /* Exact match. No other opportunity to test anything, but better than have + the macro completely untested. */ + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::Simd128})(), "SIMD128"_s); + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::Scalar})(), "scalar"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#ifdef CORRADE_TARGET_X86 +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtra(CORRADE_CPU_DECLARE(Cpu::Scalar)) { + return []() -> const char* { return "scalar"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtra(CORRADE_CPU_DECLARE(Cpu::AvxF16c|Cpu::AvxFma)) { + return []() -> const char* { return "F16C+FMA"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtra(CORRADE_CPU_DECLARE(Cpu::Avx)) { + return []() -> const char* { return "AVX"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtra(CORRADE_CPU_DECLARE(Cpu::Avx|Cpu::AvxF16c)) { + return []() -> const char* { return "AVX+F16C"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtra(CORRADE_CPU_DECLARE(Cpu::Avx2|Cpu::AvxFma)) { + return []() -> const char* { return "AVX2+FMA"; }; +} + +CORRADE_CPU_DISPATCHER(dispatchRuntimeExtra, Cpu::AvxF16c, Cpu::AvxFma) +#endif + +void CpuTest::tagDispatchRuntimeExtra() { + /* Explicitly casting to Cpu::Features here to ensure it's indeed going + through the runtime switch and not directly. Otherwise it's mostly the + same as tagDispatchExtraFallbackBoth(). */ + + #ifdef CORRADE_TARGET_X86 + /* Top-class HW, just pick AVX2 as it's the closest */ + CORRADE_COMPARE(dispatchRuntimeExtra(Cpu::Features{Cpu::Avx512f|Cpu::AvxFma|Cpu::AvxF16c})(), "AVX2+FMA"_s); + + /* We have one extra less than required for the AVX2 variant, fall back to + AVX with F16C */ + CORRADE_COMPARE(dispatchRuntimeExtra(Cpu::Features{Cpu::Avx2|Cpu::AvxF16c})(), "AVX+F16C"_s); + + /* We have AVX2, but neither of the extra bits, just some irrelevant ones, + take plain AVX */ + CORRADE_COMPARE(dispatchRuntimeExtra(Cpu::Features{Cpu::Avx2|Cpu::Lzcnt|Cpu::Popcnt})(), "AVX"_s); + + /* We have only SSE3 but both extra bits, fall back to the scalar version + that has them. Yes, it's silly, but the scalar fallback needs to be + verified. */ + CORRADE_COMPARE(dispatchRuntimeExtra(Cpu::Features{Cpu::Sse3|Cpu::AvxF16c|Cpu::AvxFma})(), "F16C+FMA"_s); + + /* Finally, this will fall back to the scalar version. */ + CORRADE_COMPARE(dispatchRuntimeExtra(Cpu::Features{Cpu::Sse3})(), "scalar"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#ifdef CORRADE_TARGET_X86 +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraCombination(CORRADE_CPU_DECLARE(Cpu::Scalar)) { + return []() -> const char* { return "scalar"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraCombination(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt)) { + return []() -> const char* { return "SSE4.1+POPCNT+LZCNT"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraCombination(CORRADE_CPU_DECLARE(Cpu::Avx2|Cpu::Popcnt|Cpu::Lzcnt)) { + return []() -> const char* { return "AVX2+POPCNT+LZCNT"; }; +} + +CORRADE_CPU_DISPATCHER(dispatchRuntimeExtraCombination, Cpu::Popcnt|Cpu::Lzcnt) +#endif + +void CpuTest::tagDispatchRuntimeExtraCombination() { + /* This verifies that the CORRADE_CPU_DISPATCHER() can accept also + tag combinations, in case the variants only ever use them together */ + + #ifdef CORRADE_TARGET_X86 + /* Verify the tag combinations don't get skipped if we have them all or + more */ + CORRADE_COMPARE(dispatchRuntimeExtraCombination(Cpu::Features{Cpu::Avx512f|Cpu::Popcnt|Cpu::Lzcnt|Cpu::AvxFma})(), "AVX2+POPCNT+LZCNT"_s); + CORRADE_COMPARE(dispatchRuntimeExtraCombination(Cpu::Features{Cpu::Avx|Cpu::Popcnt|Cpu::Lzcnt})(), "SSE4.1+POPCNT+LZCNT"_s); + + /* But also that they don't get picked if we don't have them all */ + CORRADE_COMPARE(dispatchRuntimeExtraCombination(Cpu::Features{Cpu::Avx512f|Cpu::Popcnt|Cpu::AvxFma})(), "scalar"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#if defined(CORRADE_TARGET_X86) || defined(CORRADE_TARGET_ARM) || defined(CORRADE_TARGET_WASM) +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU_DECLARE(Cpu::Scalar)) { + return []() -> const char* { return "scalar"; }; +} + +#ifdef CORRADE_TARGET_X86 +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU_DECLARE(Cpu::Sse2)) { + return []() -> const char* { return "SSE2"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU_DECLARE(Cpu::Avx)) { + return []() -> const char* { return "AVX"; }; +} +CORRADE_CPU_DISPATCHER(dispatchRuntimeExtraZeroExtra) +#elif defined(CORRADE_TARGET_ARM) +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU_DECLARE(Cpu::Neon)) { + return []() -> const char* { return "NEON"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU_DECLARE(Cpu::NeonFma)) { + return []() -> const char* { return "NEON FMA"; }; +} + +CORRADE_CPU_DISPATCHER(dispatchRuntimeExtraZeroExtra) +#elif defined(CORRADE_TARGET_WASM) +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU_DECLARE(Cpu::Simd128)) { + return []() -> const char* { return "SIMD128"; }; +} + +CORRADE_CPU_DISPATCHER(dispatchRuntimeExtraZeroExtra) +#else +#error +#endif +#endif + +void CpuTest::tagDispatchRuntimeExtraZeroExtra() { + /* Explicitly casting to Cpu::Features here to ensure it's indeed going + through the runtime switch and not directly. Otherwise it's mostly the + same as tagDispatchRuntime(). */ + + #ifdef CORRADE_TARGET_X86 + /* If no match, gets the next highest available */ + CORRADE_COMPARE(dispatchRuntimeExtraZeroExtra(Cpu::Features{Cpu::Avx512f})(), "AVX"_s); + + /* Exact match */ + CORRADE_COMPARE(dispatchRuntimeExtraZeroExtra(Cpu::Features{Cpu::Sse2})(), "SSE2"_s); + CORRADE_COMPARE(dispatchRuntimeExtraZeroExtra(Cpu::Features{Cpu::Scalar})(), "scalar"_s); + #elif defined(CORRADE_TARGET_ARM) + /* If no match, gets the next highest available */ + CORRADE_COMPARE(dispatchRuntimeExtraZeroExtra(Cpu::Features{Cpu::NeonFp16})(), "NEON FMA"_s); + + /* Exact match */ + CORRADE_COMPARE(dispatchRuntimeExtraZeroExtra(Cpu::Features{Cpu::Neon})(), "NEON"_s); + CORRADE_COMPARE(dispatchRuntimeExtraZeroExtra(Cpu::Features{Cpu::Scalar})(), "scalar"_s); + #elif defined(CORRADE_TARGET_WASM) + /* Exact match. No other opportunity to test anything, but better than have + the macro completely untested. */ + CORRADE_COMPARE(dispatchRuntimeExtraZeroExtra(Cpu::Features{Cpu::Simd128})(), "SIMD128"_s); + CORRADE_COMPARE(dispatchRuntimeExtraZeroExtra(Cpu::Features{Cpu::Scalar})(), "scalar"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +using DispatchedT = Cpu::Features(*)(); + +CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(Cpu::Scalar)) { + return []() -> Cpu::Features { return Cpu::Scalar; }; +} +#ifdef CORRADE_TARGET_X86 +CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(Cpu::Sse2)) { + return []() -> Cpu::Features { return Cpu::Sse2; }; +} +CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(Cpu::Avx)) { + return []() -> Cpu::Features { return Cpu::Avx; }; +} +CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(Cpu::Avx|Cpu::AvxFma)) { + return []() -> Cpu::Features { return Cpu::Avx|Cpu::AvxFma; }; +} + +CORRADE_CPU_DISPATCHER(dispatchedImplementation, Cpu::AvxFma) +#elif defined(CORRADE_TARGET_ARM) +CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(Cpu::NeonFp16)) { + return []() -> Cpu::Features { return Cpu::NeonFp16; }; +} +CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(Cpu::Neon)) { + return []() -> Cpu::Features { return Cpu::Neon; }; +} + +CORRADE_CPU_DISPATCHER(dispatchedImplementation) +#elif defined(CORRADE_TARGET_WASM) +CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(Cpu::Simd128)) { + return []() -> Cpu::Features { return Cpu::Simd128; }; +} + +CORRADE_CPU_DISPATCHER(dispatchedImplementation) +#endif + +/* CORRADE_NEVER_INLINE to make it possible to look at the disassembly. The + lambda body should be fully inlined here. */ +CORRADE_NEVER_INLINE Cpu::Features dispatchedCompileTime() { + /* While calling without CORRADE_CPU_SELECT() would work too, it'd go + through the runtime dispatch. And then the call would *not* get inlined, + no matter how many force inlines I put onto the lambdas. */ + return dispatchedImplementation(CORRADE_CPU_SELECT(Cpu::Default))(); +} + +void CpuTest::tagDispatchedCompileTime() { + Cpu::Features dispatchedFeatures = dispatchedCompileTime(); + CORRADE_INFO("Dispatched to:" << dispatchedFeatures); + + Cpu::Features features = Cpu::compiledFeatures(); + #ifdef CORRADE_TARGET_X86 + if(features >= (Cpu::Avx|Cpu::AvxFma)) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Avx|Cpu::AvxFma); + else if(features >= Cpu::Avx) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Avx); + else if(features >= Cpu::Sse2) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Sse2); + else + #elif defined(CORRADE_TARGET_ARM) + if(features >= Cpu::NeonFp16) + CORRADE_COMPARE(dispatchedFeatures, Cpu::NeonFp16); + else if(features >= Cpu::Neon) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Neon); + else + #elif defined(CORRADE_TARGET_WASM) + if(features >= Cpu::Simd128) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Simd128); + else + #endif + { + CORRADE_COMPARE(dispatchedFeatures, Cpu::Scalar); + } +} + +CORRADE_CPU_DISPATCHED_POINTER(dispatchedImplementation, Cpu::Features (*dispatchedPointer)()) + +void CpuTest::tagDispatchedPointer() { + Cpu::Features dispatchedFeatures = dispatchedPointer(); + CORRADE_INFO("Dispatched to:" << dispatchedFeatures); + + Cpu::Features features = Cpu::runtimeFeatures(); + #ifdef CORRADE_TARGET_X86 + if(features >= (Cpu::Avx|Cpu::AvxFma)) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Avx|Cpu::AvxFma); + else if(features >= Cpu::Avx) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Avx); + else if(features >= Cpu::Sse2) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Sse2); + else + #elif defined(CORRADE_TARGET_ARM) + if(features >= Cpu::NeonFp16) + CORRADE_COMPARE(dispatchedFeatures, Cpu::NeonFp16); + else if(features >= Cpu::Neon) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Neon); + else + #elif defined(CORRADE_TARGET_WASM) + if(features >= Cpu::Simd128) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Simd128); + else + #endif + { + CORRADE_COMPARE(dispatchedFeatures, Cpu::Scalar); + } +} + +#ifdef CORRADE_CPU_USE_IFUNC +CORRADE_CPU_DISPATCHED_IFUNC(dispatchedImplementation, Cpu::Features dispatchedIfunc()) +#endif + +void CpuTest::tagDispatchedIfunc() { + #ifndef CORRADE_CPU_USE_IFUNC + CORRADE_SKIP("CORRADE_CPU_USE_IFUNC not available"); + #else + Cpu::Features dispatchedFeatures = dispatchedIfunc(); + CORRADE_INFO("Dispatched to:" << dispatchedFeatures); + + Cpu::Features features = Cpu::runtimeFeatures(); + #ifdef CORRADE_TARGET_X86 + if(features >= (Cpu::Avx|Cpu::AvxFma)) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Avx|Cpu::AvxFma); + else if(features >= Cpu::Avx) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Avx); + else if(features >= Cpu::Sse2) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Sse2); + else + #elif defined(CORRADE_TARGET_ARM) + if(features >= Cpu::NeonFp16) + CORRADE_COMPARE(dispatchedFeatures, Cpu::NeonFp16); + else if(features >= Cpu::Neon) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Neon); + else + #elif defined(CORRADE_TARGET_WASM) + if(features >= Cpu::Simd128) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Simd128); + else + #endif + { + CORRADE_COMPARE(dispatchedFeatures, Cpu::Scalar); + } + #endif +} + +constexpr std::size_t BenchmarkDispatchedRepeats = 1000000; + +/* Otherwise Clang inlines even through the function pointer */ +#ifdef CORRADE_TARGET_CLANG +CORRADE_NEVER_INLINE +#endif +int benchmarkDispatchedImplementation(int a) { + return a + 1; +} + +CORRADE_NEVER_INLINE int benchmarkDispatchedCompileTime(int a) { + /* Because benchmarkDispatchedImplementation() is marked as never inline + on Clang, calling it from here would result in two deinlined calls, + skewing the benchmark */ + #ifdef CORRADE_TARGET_CLANG + return a + 1; + #else + return benchmarkDispatchedImplementation(a); + #endif +} + +void CpuTest::benchmarkTagDispatchedCompileTime() { + int a = 0; + CORRADE_BENCHMARK(BenchmarkDispatchedRepeats) { + a = Test::benchmarkDispatchedCompileTime(a); + } + + CORRADE_COMPARE(a, BenchmarkDispatchedRepeats); +} + +auto benchmarkDispatchedImplementation(Cpu::Features) -> int(*)(int) { + return benchmarkDispatchedImplementation; +} + +CORRADE_CPU_DISPATCHED_POINTER(benchmarkDispatchedImplementation, int(*benchmarkDispatchedPointer)(int)) + +void CpuTest::benchmarkTagDispatchedPointer() { + int a = 0; + CORRADE_BENCHMARK(BenchmarkDispatchedRepeats) { + a = Test::benchmarkDispatchedPointer(a); + } + + CORRADE_COMPARE(a, BenchmarkDispatchedRepeats); +} + +#ifdef CORRADE_CPU_USE_IFUNC +CORRADE_CPU_DISPATCHED_IFUNC(benchmarkDispatchedImplementation, int benchmarkDispatchedIfunc(int)) +#endif + +void CpuTest::benchmarkTagDispatchedIfunc() { + #ifndef CORRADE_CPU_USE_IFUNC + CORRADE_SKIP("CORRADE_CPU_USE_IFUNC not available"); + #else + int a = 0; + CORRADE_BENCHMARK(BenchmarkDispatchedRepeats) { + a = Test::benchmarkDispatchedIfunc(a); + } + + CORRADE_COMPARE(a, BenchmarkDispatchedRepeats); + #endif +} + +void CpuTest::benchmarkTagDispatchedExternalLibraryCompileTime() { + int a = 0; + CORRADE_BENCHMARK(BenchmarkDispatchedRepeats) { + a = Test::benchmarkDispatchedExternalLibraryCompileTime(a); + } + + CORRADE_COMPARE(a, BenchmarkDispatchedRepeats); +} + +void CpuTest::benchmarkTagDispatchedExternalLibraryPointer() { + int a = 0; + CORRADE_BENCHMARK(BenchmarkDispatchedRepeats) { + a = Test::benchmarkDispatchedExternalLibraryPointer(a); + } + + CORRADE_COMPARE(a, BenchmarkDispatchedRepeats); +} + +void CpuTest::benchmarkTagDispatchedExternalLibraryIfunc() { + #ifndef CORRADE_CPU_USE_IFUNC + CORRADE_SKIP("CORRADE_CPU_USE_IFUNC not available"); + #else + int a = 0; + CORRADE_BENCHMARK(BenchmarkDispatchedRepeats) { + a = Test::benchmarkDispatchedExternalLibraryIfunc(a); + } + + CORRADE_COMPARE(a, BenchmarkDispatchedRepeats); + #endif +} + +void CpuTest::benchmarkTagDispatchedExternalLibraryEveryCall() { + int a = 0; + CORRADE_BENCHMARK(BenchmarkDispatchedRepeats) { + a = Test::benchmarkDispatchedExternalLibraryEveryCall(Cpu::compiledFeatures())(a); + } + + CORRADE_COMPARE(a, BenchmarkDispatchedRepeats); +} + +/* Not using an argument here since we *don't* want the overload delegating + in this case -- it would hide errors when a certain instruction set doesn't + have a corresponding overload, as it'd fall back to a parent one. I'm also + defining a catch-all implementation with CORRADE_SKIP() instead of having an + #ifdef CORRADE_ENABLE_* around every variant in addTests(), because this way + it's clearly visible in the test output if any CORRADE_ENABLE_* macro isn't + defined for whatever reason. */ +template int callInstructionFor() { + CORRADE_SKIP("No CORRADE_ENABLE_* macro for" << Cpu::features() << "on this compiler"); +} +/* The goal here is to use instructions that would make the compilation fail + on GCC and default flags (i.e., no -march=native etc.) if the + CORRADE_ENABLE_* macro is removed. While this is quite a lot of code, it's a + good overview of how all the instructions look like... and it also uncovers + a MASSIVE amount of platform-specific warts and compiler bugs that the API + should take care of. + + All these are also marked with CORRADE_NEVER_INLINE to make it easier to see + into what code they get actually compiled. Except for the catch-all variant, + which isn't interesting for disassembly. */ +#ifdef CORRADE_ENABLE_SSE2 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(SSE2) int callInstructionFor() { + __m128i a = _mm_set_epi32(0x80808080, 0, 0x80808080, 0); + + /* All instructions SSE2 */ + + int mask = _mm_movemask_epi8(a); + CORRADE_COMPARE(mask, 0xf0f0); /* 0b1111000011110000 */ + return mask; +} +#endif +#ifdef CORRADE_ENABLE_SSE3 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(SSE3) int callInstructionFor() { + const std::uint32_t a[]{0, 10, 20, 30, 40}; + + /* SSE3 */ + union { + __m128i v; + int s[4]; + } b; + b.v = _mm_lddqu_si128(reinterpret_cast(a + 1)); + + CORRADE_COMPARE(b.s[0], 10); + CORRADE_COMPARE(b.s[1], 20); + CORRADE_COMPARE(b.s[2], 30); + CORRADE_COMPARE(b.s[3], 40); + return b.s[0]; +} +#endif +#ifdef CORRADE_ENABLE_SSSE3 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(SSSE3) int callInstructionFor() { + __m128i a = _mm_set_epi32(-10, 20, -30, 40); + + /* SSSE3 */ + union { + __m128i v; + int s[4]; + } b; + b.v = _mm_abs_epi32(a); + + CORRADE_COMPARE(b.s[3], 10); + CORRADE_COMPARE(b.s[2], 20); + CORRADE_COMPARE(b.s[1], 30); + CORRADE_COMPARE(b.s[0], 40); + return b.s[0]; +} +#endif +#ifdef CORRADE_ENABLE_SSE41 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(SSE41) int callInstructionFor() { + __m128 a = _mm_set_ps(5.47f, 2.23f, 7.62f, 0.5f); + + /* SSE4.1 */ + union { + __m128 v; + float s[4]; + } b; + b.v = _mm_ceil_ps(a); + + CORRADE_COMPARE(b.s[3], 6.0f); + CORRADE_COMPARE(b.s[2], 3.0f); + CORRADE_COMPARE(b.s[1], 8.0f); + CORRADE_COMPARE(b.s[0], 1.0f); + return b.s[0]; +} +#endif +#ifdef CORRADE_ENABLE_SSE42 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(SSE42) int callInstructionFor() { + __m128i a = _mm_set_epi64x(50, 60); + __m128i b = _mm_set_epi64x(60, 50); + + /* SSE4.2 */ + union { + __m128i v; + std::int64_t s[2]; + } c; + c.v = _mm_cmpgt_epi64(a, b); + + CORRADE_COMPARE(c.s[0], -1); + CORRADE_COMPARE(c.s[1], 0); + return c.s[0]; +} +#endif +#ifdef CORRADE_ENABLE_POPCNT +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(POPCNT) int callInstructionFor() { + /* Just pocnt alone; using volatile to prevent this from being folded into + a constant */ + volatile unsigned int a = 0x0005c1a6; + unsigned int count = _mm_popcnt_u32(a); + CORRADE_COMPARE(count, 9); + return count; +} +#endif +#ifdef CORRADE_ENABLE_LZCNT +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(LZCNT) int callInstructionFor() { + /* Just lzcnt alone; using volatile to prevent this from being folded into + a constant */ + volatile int a = 0x0005c1a6; + unsigned int count = _lzcnt_u32(a); + + /* Also verify that it does the right thing for 0. If misdetected and the + BSR fallback gets used, this would return something random here. */ + CORRADE_COMPARE(_lzcnt_u32(0), 32); + + CORRADE_COMPARE(count, 13); + return count; +} +#endif +#ifdef CORRADE_ENABLE_BMI1 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(BMI1) int callInstructionFor() { + /* Just lzcnt alone; using volatile to prevent this from being folded into + a constant */ + volatile int a = 0x6583a000; /* 0x0005c1a6 but bit-reversed */ + /* GCC and Clang have __tzcnt_u32() as well, but MSVC has only a single + underscore. */ + unsigned int count = _tzcnt_u32(a); + + /* Also verify that it does the right thing for 0. If misdetected and the + BSR fallback gets used, this would return something random here. */ + CORRADE_COMPARE(_tzcnt_u32(0), 32); + + CORRADE_COMPARE(count, 13); + return count; +} +#endif +#ifdef CORRADE_ENABLE_AVX +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(AVX) int callInstructionFor() { + __m256d a = _mm256_set_pd(5.47, 2.23, 7.62, 0.5); + + /* All instructions AVX */ + + union { + __m256d v; + double s[4]; + } b; + b.v = _mm256_ceil_pd(a); + + CORRADE_COMPARE(b.s[3], 6.0); + CORRADE_COMPARE(b.s[2], 3.0); + CORRADE_COMPARE(b.s[1], 8.0); + CORRADE_COMPARE(b.s[0], 1.0); + return b.s[0]; +} +#endif +#ifdef CORRADE_ENABLE_AVX_F16C +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(AVX_F16C) int callInstructionFor() { + /* Values from Magnum::Math::Test::HalfTest::pack() */ + __m128 a = _mm_set_ps(0.0f, 123.75f, -0.000351512f, 3.0f); + + /* F16C */ + union { + __m128i v; + std::uint16_t s[8]; + } b; + b.v = _mm_cvtps_ph(a, 0); + + CORRADE_COMPARE(b.s[3], 0x0000); + CORRADE_COMPARE(b.s[2], 0x57bc); + CORRADE_COMPARE(b.s[1], 0x8dc2); + CORRADE_COMPARE(b.s[0], 0x4200); + return b.s[0]; +} +#endif +#ifdef CORRADE_ENABLE_AVX_FMA +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(AVX_FMA) int callInstructionFor() { + /* Values from Magnum::Math::Test::FunctionsTest::fma() */ + __m128 a = _mm_set_ps(0.0f, 2.0f, 1.5f, 0.5f); + __m128 b = _mm_set_ps(0.0f, 3.0f, 2.0f, -1.0f); + __m128 c = _mm_set_ps(0.0f, 0.75f, 0.25f, 0.1f); + + /* FMA */ + union { + __m128 v; + float s[4]; + } d; + d.v = _mm_fmadd_ps(a, b, c); + + CORRADE_COMPARE(d.s[3], 0.0f); + CORRADE_COMPARE(d.s[2], 6.75f); + CORRADE_COMPARE(d.s[1], 3.25f); + CORRADE_COMPARE(d.s[0], -0.4f); + return d.s[2]; +} +#endif +#ifdef CORRADE_ENABLE_AVX2 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(AVX2) int callInstructionFor() { + __m256i a = _mm256_set_epi64x(0x8080808080808080ull, 0, 0x8080808080808080ull, 0); + + /* Like callInstructionFor(), but expanded to AVX2 */ + int mask = _mm256_movemask_epi8(a); + + CORRADE_COMPARE(mask, 0xff00ff00);/* 0b11111111000000001111111100000000 */ + return mask; +} +#endif +#ifdef CORRADE_ENABLE_AVX512F +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(AVX512F) int callInstructionFor() { + __m128 a = _mm_set1_ps(5.47f); + + /* AVX512 */ + int ceil = _mm_cvt_roundss_si32(a, _MM_FROUND_TO_POS_INF|_MM_FROUND_NO_EXC); + + CORRADE_COMPARE(ceil, 6); + return ceil; +} +#endif +#ifdef CORRADE_ENABLE_NEON +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(NEON) int callInstructionFor() { + int32x4_t a{-10, 20, -30, 40}; + + /* All instructions NEON */ + union { + int32x4_t v; + int s[8]; + } b; + b.v = vabsq_s32(a); + + CORRADE_COMPARE(b.s[0], 10); + CORRADE_COMPARE(b.s[1], 20); + CORRADE_COMPARE(b.s[2], 30); + CORRADE_COMPARE(b.s[3], 40); + return b.s[0]; +} +#endif +#ifdef CORRADE_ENABLE_NEON_FMA +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(NEON_FMA) int callInstructionFor() { + /* Values from Magnum::Math::Test::FunctionsTest::fma() */ + float32x4_t a{0.0f, 2.0f, 1.5f, 0.5f}; + float32x4_t b{0.0f, 3.0f, 2.0f, -1.0f}; + float32x4_t c{0.0f, 0.75f, 0.25f, 0.1f}; + + /* FMA */ + union { + float32x4_t v; + float s[4]; + } d; + d.v = vfmaq_f32(c, b, a); + + CORRADE_COMPARE(d.s[0], 0.0f); + CORRADE_COMPARE(d.s[1], 6.75f); + CORRADE_COMPARE(d.s[2], 3.25f); + CORRADE_COMPARE(d.s[3], -0.4f); + return d.s[2]; +} +#endif +#ifdef CORRADE_ENABLE_NEON_FP16 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(NEON_FP16) int callInstructionFor() { + float32x4_t a{5.47f, 2.23f, 7.62f, 0.5f}; + float16x4_t b = vcvt_f16_f32(a); + + /* FP16 */ + float16x4_t c = vrndp_f16(b); + + union { + float32x4_t v; + float s[4]; + } d; + d.v = vcvt_f32_f16(c); + CORRADE_COMPARE(d.s[0], 6.0f); + CORRADE_COMPARE(d.s[1], 3.0f); + CORRADE_COMPARE(d.s[2], 8.0f); + CORRADE_COMPARE(d.s[3], 1.0f); + return d.s[3]; +} +#endif +#ifdef CORRADE_ENABLE_SIMD128 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(SIMD128) int callInstructionFor() { + v128_t a = wasm_f32x4_make(5.47, 2.23, 7.62, 0.5); + + /* All instructions SIMD128. wasm_f32x4_ceil() is only available in the + finalized wasm intrinsics that's since Clang 13 which is used since + Emscripten 2.0.13, thus any older version should not have the + CORRADE_ENABLE_SIMD128 macro defined: + https://github.com/llvm/llvm-project/commit/502f54049d17f5a107f833596fb2c31297a99773 + https://github.com/emscripten-core/emscripten/commit/deab7783df407b260f46352ffad2a77ca8fb0a4c */ + union { + v128_t v; + float s[8]; + } b; + b.v = wasm_f32x4_ceil(a); + + CORRADE_COMPARE(b.s[0], 6.0); + CORRADE_COMPARE(b.s[1], 3.0); + CORRADE_COMPARE(b.s[2], 8.0); + CORRADE_COMPARE(b.s[3], 1.0); + return b.s[0]; +} +#endif + +template void CpuTest::enableMacros() { + setTestCaseTemplateName(Cpu::TypeTraits::name()); + + if(!(Cpu::runtimeFeatures() & Cpu::features())) + CORRADE_SKIP("CPU feature not supported"); + + CORRADE_VERIFY(true); /* to capture correct function name */ + CORRADE_VERIFY(callInstructionFor()); +} + +#if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_AVX2) && defined(CORRADE_ENABLE_AVX) +/* If set to 1, should fail on GCC (unless CORRADE_TARGET_AVX is set) */ +#if 0 +CORRADE_ENABLE_AVX2 CORRADE_ENABLE_AVX +#else +/* If CORRADE_TARGET_SSE2 is set (on 64bit), it'll result in just "avx2,avx" */ +CORRADE_ENABLE(AVX2,SSE2,AVX) +#endif +int callInstructionMultiple() { + if(!(Cpu::runtimeFeatures() & Cpu::Avx2)) + CORRADE_SKIP("AVX2 feature not supported"); + + /* Same as callInstructionFor() */ + __m256i a = _mm256_set_epi64x(0x8080808080808080ull, 0, 0x8080808080808080ull, 0); + + /* If the AVX2 instructions aren't enabled, this will fail to link */ + int mask = _mm256_movemask_epi8(a); + + CORRADE_COMPARE(mask, 0xff00ff00); + return mask; +} +#elif defined(CORRADE_ENABLE_NEON_FMA) && defined(CORRADE_ENABLE_NEON) +/* If set to 1, should fail on GCC (unless CORRADE_TARGET_NEON is set) */ +#if 0 +CORRADE_ENABLE_NEON_FMA CORRADE_ENABLE_NEON +#else +CORRADE_ENABLE(NEON_FMA,NEON) +#endif +int callInstructionMultiple() { + if(!(Cpu::runtimeFeatures() & Cpu::NeonFma)) + CORRADE_SKIP("NEON FMA feature not supported"); + + /* Same as callInstructionFor() */ + float32x4_t a{0.0f, 2.0f, 1.5f, 0.5f}; + float32x4_t b{0.0f, 3.0f, 2.0f, -1.0f}; + float32x4_t c{0.0f, 0.75f, 0.25f, 0.1f}; + + union { + float32x4_t v; + float s[4]; + } d; + /* If the FMA instructions aren't enabled, this will fail to link */ + d.v = vfmaq_f32(c, b, a); + + CORRADE_COMPARE(d.s[0], 0.0f); + CORRADE_COMPARE(d.s[1], 6.75f); + CORRADE_COMPARE(d.s[2], 3.25f); + CORRADE_COMPARE(d.s[3], -0.4f); + return d.s[2]; +} +#else +int callInstructionMultiple() { + CORRADE_SKIP("Not enough CORRADE_ENABLE_ macros defined"); +} +#endif + +void CpuTest::enableMacrosMultiple() { + CORRADE_VERIFY(callInstructionMultiple()); +} + +/* If the ENABLE_ macro is empty, it should not result in any __attribute__ + annotation */ +#ifdef CORRADE_TARGET_SSE2 +CORRADE_ENABLE(SSE2) int callInstructionMultipleAllEmpty() { + return 1; +} +#elif defined(CORRADE_TARGET_NEON) +CORRADE_ENABLE(NEON) int callInstructionMultipleAllEmpty() { + return 1; +} +#elif defined(CORRADE_TARGET_SIMD128) +CORRADE_ENABLE(SIMD128) int callInstructionMultipleAllEmpty() { + return 1; +} +#else +int callInstructionMultipleAllEmpty() { + CORRADE_SKIP("No suitable CORRADE_TARGET_ macro defined"); +} +#endif + +void CpuTest::enableMacrosMultipleAllEmpty() { + CORRADE_COMPARE(callInstructionMultipleAllEmpty(), 1); +} + +/* On Clang it's enough to have the ENABLE macro just on the wrapper function. + On GCC it has to be attached to the lambda due to + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80439. However, older versions + of Clang suffer from the inverse problem and ignore lambda attributes, so it + has to stay on the function definition as well. Furthermore, if the trailing + return type is uncommented, the code will fail to compile on GCC 9.1 to 9.3: + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90333 */ +#ifdef CORRADE_ENABLE_AVX2 +CORRADE_ENABLE_AVX2 int callInstructionLambda() { + return []() CORRADE_ENABLE_AVX2 /*-> int*/ { + if(!(Cpu::runtimeFeatures() & Cpu::Avx2)) + CORRADE_SKIP("AVX2 feature not supported"); + + /* Same as callInstructionFor() */ + __m256i a = _mm256_set_epi64x(0x8080808080808080ull, 0, 0x8080808080808080ull, 0); + + /* If the AVX2 instructions aren't enabled, this will fail to link */ + int mask = _mm256_movemask_epi8(a); + + CORRADE_COMPARE(mask, 0xff00ff00); + return mask; + }(); +} +#elif defined(CORRADE_ENABLE_NEON) +CORRADE_ENABLE_NEON int callInstructionLambda() { + return []() CORRADE_ENABLE_NEON /*-> int*/ { + if(!(Cpu::runtimeFeatures() & Cpu::Neon)) + CORRADE_SKIP("NEON feature not supported"); + + /* Same as callInstructionFor() */ + int32x4_t a{-10, 20, -30, 40}; + + union { + int32x4_t v; + int s[8]; + } b; + b.v = vabsq_s32(a); + + CORRADE_COMPARE(b.s[0], 10); + CORRADE_COMPARE(b.s[1], 20); + CORRADE_COMPARE(b.s[2], 30); + CORRADE_COMPARE(b.s[3], 40); + return b.s[0]; + }(); +} +#elif defined(CORRADE_ENABLE_SIMD128) +CORRADE_ENABLE_SIMD128 int callInstructionLambda() { + return []() CORRADE_ENABLE_SIMD128 /*-> int*/ { + if(!(Cpu::runtimeFeatures() & Cpu::Simd128)) + CORRADE_SKIP("SIMD128 feature not supported"); + + /* Same as callInstructionFor() */ + v128_t a = wasm_f32x4_make(5.47, 2.23, 7.62, 0.5); + + union { + v128_t v; + float s[8]; + } b; + b.v = wasm_f32x4_ceil(a); + + CORRADE_COMPARE(b.s[0], 6.0); + CORRADE_COMPARE(b.s[1], 3.0); + CORRADE_COMPARE(b.s[2], 8.0); + CORRADE_COMPARE(b.s[3], 1.0); + return b.s[0]; + }(); +} +#else +int callInstructionLambda() { + CORRADE_SKIP("No usable CORRADE_ENABLE_ macros defined"); +} +#endif + +void CpuTest::enableMacrosLambda() { + /* Verifies that CORRADE_ENABLE_* can be applied also to lambdas. See the + comment above the implementations for more information. */ + CORRADE_VERIFY(callInstructionLambda()); +} + +/* Same as callInstructionLambda(), just with CORRADE_ENABLE(*) instead of + CORRADE_ENABLE_* */ +#ifdef CORRADE_ENABLE_AVX2 +CORRADE_ENABLE(AVX2) int callInstructionLambdaMultiple() { + return []() CORRADE_ENABLE(AVX2) /*-> int*/ { + if(!(Cpu::runtimeFeatures() & Cpu::Avx2)) + CORRADE_SKIP("AVX2 feature not supported"); + + /* Same as callInstructionFor() */ + __m256i a = _mm256_set_epi64x(0x8080808080808080ull, 0, 0x8080808080808080ull, 0); + + /* If the AVX2 instructions aren't enabled, this will fail to link */ + int mask = _mm256_movemask_epi8(a); + + CORRADE_COMPARE(mask, 0xff00ff00); + return mask; + }(); +} +#elif defined(CORRADE_ENABLE_NEON) +CORRADE_ENABLE(NEON) int callInstructionLambdaMultiple() { + return []() CORRADE_ENABLE(NEON) /*-> int*/ { + if(!(Cpu::runtimeFeatures() & Cpu::Neon)) + CORRADE_SKIP("NEON feature not supported"); + + /* Same as callInstructionFor() */ + int32x4_t a{-10, 20, -30, 40}; + + union { + int32x4_t v; + int s[8]; + } b; + b.v = vabsq_s32(a); + + CORRADE_COMPARE(b.s[0], 10); + CORRADE_COMPARE(b.s[1], 20); + CORRADE_COMPARE(b.s[2], 30); + CORRADE_COMPARE(b.s[3], 40); + return b.s[0]; + }(); +} +#elif defined(CORRADE_ENABLE_SIMD128) +CORRADE_ENABLE(SIMD128) int callInstructionLambdaMultiple() { + return []() CORRADE_ENABLE(SIMD128) /*-> int*/ { + if(!(Cpu::runtimeFeatures() & Cpu::Simd128)) + CORRADE_SKIP("SIMD128 feature not supported"); + + /* Same as callInstructionFor() */ + v128_t a = wasm_f32x4_make(5.47, 2.23, 7.62, 0.5); + + union { + v128_t v; + float s[8]; + } b; + b.v = wasm_f32x4_ceil(a); + + CORRADE_COMPARE(b.s[0], 6.0); + CORRADE_COMPARE(b.s[1], 3.0); + CORRADE_COMPARE(b.s[2], 8.0); + CORRADE_COMPARE(b.s[3], 1.0); + return b.s[0]; + }(); +} +#else +int callInstructionLambdaMultiple() { + CORRADE_SKIP("No usable CORRADE_ENABLE_ macros defined"); +} +#endif + +void CpuTest::enableMacrosLambdaMultiple() { + /* Verifies that CORRADE_ENABLE(*) (the function macro variant) can be + applied to lambdas as well */ + CORRADE_VERIFY(callInstructionLambdaMultiple()); +} + +void CpuTest::debug() { + /* If more flags get added, this might need to become even more zeros */ + const unsigned int dead = 0xde00ad00; + + /* Features{} are equivalent to Scalar */ + #ifdef CORRADE_TARGET_X86 + std::ostringstream out; + Debug{&out} << Cpu::Scalar << (Cpu::Avx2|Cpu::Ssse3|Cpu::Sse41) << Cpu::Features{} << (reinterpret_cast(dead)|Cpu::Avx512f) << reinterpret_cast(dead); + CORRADE_COMPARE(out.str(), "Cpu::Scalar Cpu::Ssse3|Cpu::Sse41|Cpu::Avx2 Cpu::Scalar Cpu::Avx512f|Cpu::Features(0xde00ad00) Cpu::Features(0xde00ad00)\n"); + #elif defined(CORRADE_TARGET_ARM) + std::ostringstream out; + Debug{&out} << Cpu::Scalar << (Cpu::NeonFp16|Cpu::NeonFma|Cpu::Neon) << Cpu::Features{} << (reinterpret_cast(dead)|Cpu::NeonFma) << reinterpret_cast(dead); + CORRADE_COMPARE(out.str(), "Cpu::Scalar Cpu::Neon|Cpu::NeonFma|Cpu::NeonFp16 Cpu::Scalar Cpu::NeonFma|Cpu::Features(0xde00ad00) Cpu::Features(0xde00ad00)\n"); + #else + static_cast(dead); + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +void CpuTest::debugPacked() { + /* If more flags get added, this might need to become even more zeros */ + const unsigned int dead = 0xde00ad00; + + /* Features{} are equivalent to Scalar */ + #ifdef CORRADE_TARGET_X86 + std::ostringstream out; + Debug{&out} << Debug::packed << Cpu::Scalar << Debug::packed << (Cpu::Avx2|Cpu::Ssse3|Cpu::Sse41) << Debug::packed << Cpu::Features{} << Debug::packed << (reinterpret_cast(dead)|Cpu::Avx512f) << Debug::packed << reinterpret_cast(dead) << Cpu::Avx; + CORRADE_COMPARE(out.str(), "Scalar Ssse3|Sse41|Avx2 Scalar Avx512f|0xde00ad00 0xde00ad00 Cpu::Avx\n"); + #elif defined(CORRADE_TARGET_ARM) + std::ostringstream out; + Debug{&out} << Debug::packed << Cpu::Scalar << Debug::packed << (Cpu::NeonFp16|Cpu::NeonFma|Cpu::Neon) << Debug::packed << Cpu::Features{} << Debug::packed << (reinterpret_cast(dead)|Cpu::NeonFma) << Debug::packed << reinterpret_cast(dead) << Cpu::NeonFma; + CORRADE_COMPARE(out.str(), "Scalar Neon|NeonFma|NeonFp16 Scalar NeonFma|0xde00ad00 0xde00ad00 Cpu::NeonFma\n"); + #else + static_cast(dead); + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +}}} + +CORRADE_TEST_MAIN(Corrade::Test::CpuTest) diff --git a/src/Corrade/Test/CpuTestExternalLibrary.cpp b/src/Corrade/Test/CpuTestExternalLibrary.cpp new file mode 100644 index 000000000..8a3f5fe76 --- /dev/null +++ b/src/Corrade/Test/CpuTestExternalLibrary.cpp @@ -0,0 +1,55 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include "CpuTestExternalLibrary.h" + +#include "Corrade/Cpu.h" + +namespace Corrade { namespace Test { + +namespace { + +CORRADE_ALWAYS_INLINE auto implementation(Cpu::Features) -> int(*)(int) { + return [](int a) { return a + 1; }; +} + +} + +int benchmarkDispatchedExternalLibraryCompileTime(int a) { + return implementation(Cpu::DefaultBase)(a); +} + +CORRADE_CPU_DISPATCHED_POINTER(implementation, int (*benchmarkDispatchedExternalLibraryPointer)(int)) + +#ifdef CORRADE_CPU_USE_IFUNC +CORRADE_CPU_DISPATCHED_IFUNC(implementation, int benchmarkDispatchedExternalLibraryIfunc(int)) +#endif + +auto benchmarkDispatchedExternalLibraryEveryCall(Cpu::Features features) -> int(*)(int) { + return implementation(features); +} + +}} diff --git a/src/Corrade/Test/CpuTestExternalLibrary.h b/src/Corrade/Test/CpuTestExternalLibrary.h new file mode 100644 index 000000000..7ebad2d9b --- /dev/null +++ b/src/Corrade/Test/CpuTestExternalLibrary.h @@ -0,0 +1,52 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include "Corrade/Corrade.h" +#include "Corrade/Utility/VisibilityMacros.h" + +#ifndef CORRADE_PLUGINMANAGER_NO_DYNAMIC_PLUGIN_SUPPORT + #ifdef CpuTestExternalLibrary_EXPORTS + #define CORRADE_CPU_TEST_EXPORT CORRADE_VISIBILITY_EXPORT + #else + #define CORRADE_CPU_TEST_EXPORT CORRADE_VISIBILITY_IMPORT + #endif +#else + #define CORRADE_CPU_TEST_EXPORT CORRADE_VISIBILITY_STATIC +#endif + +namespace Corrade { namespace Test { + +CORRADE_CPU_TEST_EXPORT int benchmarkDispatchedExternalLibraryCompileTime(int); + +CORRADE_CPU_TEST_EXPORT extern int(*benchmarkDispatchedExternalLibraryPointer)(int); + +#ifdef CORRADE_CPU_USE_IFUNC +CORRADE_CPU_TEST_EXPORT int benchmarkDispatchedExternalLibraryIfunc(int); +#endif + +CORRADE_CPU_TEST_EXPORT auto benchmarkDispatchedExternalLibraryEveryCall(Cpu::Features) -> int(*)(int); + +}} diff --git a/src/Corrade/Test/TargetTest.cpp b/src/Corrade/Test/TargetTest.cpp index 1baf5ff77..b66455f91 100644 --- a/src/Corrade/Test/TargetTest.cpp +++ b/src/Corrade/Test/TargetTest.cpp @@ -354,17 +354,17 @@ void TargetTest::simd() { Debug{&out} << "CORRADE_TARGET_NEON"; #endif - #ifdef CORRADE_TARGET_NEON_FP16 - Debug{&out} << "CORRADE_TARGET_NEON_FP16"; + #ifdef CORRADE_TARGET_NEON_FMA + Debug{&out} << "CORRADE_TARGET_NEON_FMA"; #ifndef CORRADE_TARGET_NEON - CORRADE_VERIFY(!"CORRADE_TARGET_NEON_FP16 defined but CORRADE_TARGET_NEON not"); + CORRADE_VERIFY(!"CORRADE_TARGET_NEON_FMA defined but CORRADE_TARGET_NEON not"); #endif #endif - #ifdef CORRADE_TARGET_NEON_FMA - Debug{&out} << "CORRADE_TARGET_NEON_FMA"; - #ifndef CORRADE_TARGET_NEON_FP16 - CORRADE_VERIFY(!"CORRADE_TARGET_NEON_FMA defined but CORRADE_TARGET_NEON_FP16 not"); + #ifdef CORRADE_TARGET_NEON_FP16 + Debug{&out} << "CORRADE_TARGET_NEON_FP16"; + #ifndef CORRADE_TARGET_NEON_FMA + CORRADE_VERIFY(!"CORRADE_TARGET_NEON_FP16 defined but CORRADE_TARGET_NEON_FMA not"); #endif #endif @@ -372,12 +372,12 @@ void TargetTest::simd() { CORRADE_VERIFY(!"CORRADE_TARGET_NEON* defined but CORRADE_TARGET_ARM not"); #endif - #ifdef CORRADE_TARGET_EMSCRIPTEN + #ifdef CORRADE_TARGET_WASM #ifdef CORRADE_TARGET_SIMD128 Debug{&out} << "CORRADE_TARGET_SIMD128"; #endif #elif defined(CORRADE_TARGET_SIMD128) - CORRADE_VERIFY(!"CORRADE_TARGET_SIMD128 defined but CORRADE_TARGET_EMSCRIPTEN not"); + CORRADE_VERIFY(!"CORRADE_TARGET_SIMD128 defined but CORRADE_TARGET_WASM not"); #endif Debug{Debug::Flag::NoNewlineAtTheEnd} << out.str(); diff --git a/src/Corrade/TestSuite/CMakeLists.txt b/src/Corrade/TestSuite/CMakeLists.txt index e6670e0dd..0b31ca109 100644 --- a/src/Corrade/TestSuite/CMakeLists.txt +++ b/src/Corrade/TestSuite/CMakeLists.txt @@ -48,19 +48,18 @@ set(CorradeTestSuite_PRIVATE_HEADERS Implementation/BenchmarkCounters.h Implementation/BenchmarkStats.h) -# TestSuite library -add_library(CorradeTestSuite ${SHARED_OR_STATIC} +# Objects shared between main and test library +add_library(CorradeTestSuiteObjects OBJECT ${CorradeTestSuite_SRCS} ${CorradeTestSuite_HEADERS} ${CorradeTestSuite_PRIVATE_HEADERS}) -set_target_properties(CorradeTestSuite PROPERTIES - DEBUG_POSTFIX "-d") +target_include_directories(CorradeTestSuiteObjects PUBLIC $) if(NOT CORRADE_BUILD_STATIC) - set_target_properties(CorradeTestSuite PROPERTIES VERSION ${CORRADE_LIBRARY_VERSION} SOVERSION ${CORRADE_LIBRARY_SOVERSION}) -elseif(CORRADE_BUILD_STATIC_PIC) - set_target_properties(CorradeTestSuite PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_compile_definitions(CorradeTestSuiteObjects PRIVATE "-DCorradeTestSuiteObjects_EXPORTS") +endif() +if(NOT CORRADE_BUILD_STATIC OR CORRADE_BUILD_STATIC_PIC) + set_target_properties(CorradeTestSuiteObjects PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() -target_link_libraries(CorradeTestSuite CorradeUtility) if(CORRADE_TARGET_EMSCRIPTEN) # Since (probably) 1.38.36, the compiler needs to have exceptions enabled # as well (not just the linker), and directly for the TestSuite library. @@ -69,9 +68,22 @@ if(CORRADE_TARGET_EMSCRIPTEN) # so doing it the old way --- and instead of the options being transitively # passed to the target, for the test target it's done explicitly inside # UseCorrade.cmake. Wasn't needed with 1.38.32. - set_property(TARGET CorradeTestSuite APPEND_STRING PROPERTY COMPILE_FLAGS " -s DISABLE_EXCEPTION_CATCHING=0") + set_property(TARGET CorradeTestSuiteObjects APPEND_STRING PROPERTY COMPILE_FLAGS " -s DISABLE_EXCEPTION_CATCHING=0") endif() +# Main TestSuite library +add_library(CorradeTestSuite ${SHARED_OR_STATIC} + $ + ${PROJECT_SOURCE_DIR}/src/dummy.cpp) # XCode workaround, see file comment for details +set_target_properties(CorradeTestSuite PROPERTIES + DEBUG_POSTFIX "-d") +if(NOT CORRADE_BUILD_STATIC) + set_target_properties(CorradeTestSuite PROPERTIES VERSION ${CORRADE_LIBRARY_VERSION} SOVERSION ${CORRADE_LIBRARY_SOVERSION}) +elseif(CORRADE_BUILD_STATIC_PIC) + set_target_properties(CorradeTestSuite PROPERTIES POSITION_INDEPENDENT_CODE ON) +endif() +target_link_libraries(CorradeTestSuite PUBLIC CorradeUtility) + install(TARGETS CorradeTestSuite RUNTIME DESTINATION ${CORRADE_BINARY_INSTALL_DIR} LIBRARY DESTINATION ${CORRADE_LIBRARY_INSTALL_DIR} @@ -88,6 +100,23 @@ endif() add_subdirectory(Compare) if(CORRADE_BUILD_TESTS) + # Library that links against CorradeUtilityTestLib instead of + # CorradeUtility for testing. No other difference. Not actually used by any + # TestSuite tests, but a special case needed only by the Utility library + # -- since TestSuite depends on Utility, Utility tests that would link to + # CorradeTestSuite would get CorradeUtility in addition to + # CorradeUtilityTestLib, leading to ODR violations and subsequently ASan + # complaints. + add_library(CorradeTestSuiteTestLib ${SHARED_OR_STATIC} + $ + ${PROJECT_SOURCE_DIR}/src/dummy.cpp) # XCode workaround, see file comment for details + target_include_directories(CorradeTestSuiteTestLib PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + set_target_properties(CorradeTestSuiteTestLib PROPERTIES DEBUG_POSTFIX "-d") + if(CORRADE_BUILD_STATIC_PIC) + set_target_properties(CorradeTestSuiteTestLib PROPERTIES POSITION_INDEPENDENT_CODE ON) + endif() + target_link_libraries(CorradeTestSuiteTestLib PUBLIC CorradeUtilityTestLib) + add_subdirectory(Test) endif() diff --git a/src/Corrade/TestSuite/visibility.h b/src/Corrade/TestSuite/visibility.h index 4c0d90e0a..d7fd28d87 100644 --- a/src/Corrade/TestSuite/visibility.h +++ b/src/Corrade/TestSuite/visibility.h @@ -31,7 +31,7 @@ #ifndef DOXYGEN_GENERATING_OUTPUT #ifndef CORRADE_BUILD_STATIC - #ifdef CorradeTestSuite_EXPORTS + #if defined(CorradeTestSuite_EXPORTS) || defined(CorradeTestSuiteObjects_EXPORTS) || defined(CorradeTestSuiteTestLib_EXPORTS) #define CORRADE_TESTSUITE_EXPORT CORRADE_VISIBILITY_EXPORT #else #define CORRADE_TESTSUITE_EXPORT CORRADE_VISIBILITY_IMPORT diff --git a/src/Corrade/Utility/CMakeLists.txt b/src/Corrade/Utility/CMakeLists.txt index 24c6714ab..1d6762a15 100644 --- a/src/Corrade/Utility/CMakeLists.txt +++ b/src/Corrade/Utility/CMakeLists.txt @@ -34,10 +34,11 @@ if(CORRADE_WITH_UTILITY) Configuration.cpp ConfigurationValue.cpp MurmurHash2.cpp - Path.cpp Sha1.cpp System.cpp + ../Cpu.cpp + Implementation/ErrorString.cpp) set(CorradeUtility_GracefulAssert_SRCS @@ -56,6 +57,13 @@ if(CORRADE_WITH_UTILITY) ../Containers/String.cpp ../Containers/StringView.cpp) + # Files that directly or indirectly use CPU dispatch, such as calling + # Containers::String::contains(). The list gets added to either + # CorradeUtility_GracefulAssert_SRCS or CorradeUtility_SRCS below based on + # whether tests use a different CPU dispatch than the main library. + set(CorradeUtility_CpuDispatch_SRCS + Path.cpp) + set(CorradeUtility_HEADERS Algorithms.h Arguments.h @@ -73,6 +81,19 @@ if(CORRADE_WITH_UTILITY) Format.h FormatStl.h FormatStlStringView.h + + # Because various platforms can have multi-architecture binaries + # (macOS, iOS), the CORRADE_TARGET_X86 variable isn't exposed to CMake. + # And because all desktop and mobile platfoms have an x86 variant + # including iOS and Android, and Emscripten has x86 intrinsics + # compatibility (!!), we install the x86-specific intrinsics wrappers + # always. + IntrinsicsSse2.h + IntrinsicsSse3.h + IntrinsicsSsse3.h + IntrinsicsSse4.h + IntrinsicsAvx.h + Json.h JsonWriter.h Macros.h @@ -112,7 +133,8 @@ if(CORRADE_WITH_UTILITY) if(CORRADE_TARGET_UNIX OR (CORRADE_TARGET_WINDOWS AND NOT CORRADE_TARGET_WINDOWS_RT) OR CORRADE_TARGET_EMSCRIPTEN) list(APPEND CorradeUtility_SRCS FileWatcher.cpp - Tweakable.cpp + Tweakable.cpp) + list(APPEND CorradeUtility_CpuDispatch_SRCS TweakableParser.cpp) list(APPEND CorradeUtility_HEADERS FileWatcher.h @@ -134,6 +156,16 @@ if(CORRADE_WITH_UTILITY) list(APPEND CorradeUtility_PRIVATE_HEADERS Implementation/WindowsWeakSymbol.h) endif() + # If the tests are built with pointer dispatch but the main libraries + # aren't, we have to compile select files separately for each. Otherwise + # some code would expect a function symbol to be exist and some expect a + # function pointer symbol. + if(CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH AND (NOT CORRADE_BUILD_CPU_RUNTIME_DISPATCH OR CORRADE_CPU_USE_IFUNC)) + list(APPEND CorradeUtility_GracefulAssert_SRCS ${CorradeUtility_CpuDispatch_SRCS}) + else() + list(APPEND CorradeUtility_SRCS ${CorradeUtility_CpuDispatch_SRCS}) + endif() + # Objects shared between main and test library add_library(CorradeUtilityObjects OBJECT ${CorradeUtility_SRCS} @@ -182,16 +214,31 @@ if(CORRADE_WITH_UTILITY) install(FILES ${CorradeUtility_HEADERS} DESTINATION ${CORRADE_INCLUDE_INSTALL_DIR}/Utility) if(CORRADE_BUILD_TESTS) - # Library with graceful assert for testing - add_library(CorradeUtilityTestLib ${SHARED_OR_STATIC} ${CorradeUtility_GracefulAssert_SRCS}) + # Library with graceful assert, function-pointer-based CPU dispatch + # and WASM SIMD128 for testing. If tests need such behavior, they + # should link to CorradeTestSuiteTestLib instead. Otherwise the + # implicitly linked CorradeTestSuite would drag in CorradeUtility in + # addition to CorradeUtilityTestLib, leading to ODR violations and + # making ASan builds fail. + add_library(CorradeUtilityTestLib ${SHARED_OR_STATIC} + $ + ${CorradeUtility_GracefulAssert_SRCS}) + target_include_directories(CorradeUtilityTestLib PUBLIC + $) + target_link_libraries(CorradeUtilityTestLib PUBLIC + $) target_compile_definitions(CorradeUtilityTestLib PRIVATE "CORRADE_GRACEFUL_ASSERT") + if(CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH) + target_compile_definitions(CorradeUtilityTestLib PUBLIC "CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH") + if(CORRADE_BUILD_TESTS_FORCE_WASM_SIMD128) + target_compile_options(CorradeUtilityTestLib PUBLIC "-msimd128") + endif() + endif() set_target_properties(CorradeUtilityTestLib PROPERTIES DEBUG_POSTFIX "-d") if(CORRADE_BUILD_STATIC_PIC) set_target_properties(CorradeUtilityTestLib PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() - target_link_libraries(CorradeUtilityTestLib CorradeUtility) - add_subdirectory(Test) endif() @@ -220,6 +267,7 @@ if(NOT CMAKE_CROSSCOMPILING) Implementation/ErrorString.cpp + ../Cpu.cpp ../Containers/String.cpp ../Containers/StringView.cpp) if(CORRADE_TARGET_WINDOWS) diff --git a/src/Corrade/Utility/IntrinsicsAvx.h b/src/Corrade/Utility/IntrinsicsAvx.h new file mode 100644 index 000000000..1f27acc3b --- /dev/null +++ b/src/Corrade/Utility/IntrinsicsAvx.h @@ -0,0 +1,95 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +/** @file +@brief Intrinsics for x86 LZCNT, BMI1, AVX, AVX F16C, AVX FMA and AVX2 instructions +@m_since_latest + +Equivalent to @cpp #include @ce on most compilers except for GCC +4.8, where it contains an additional workaround to make the instructions +available with just the @ref CORRADE_ENABLE_AVX, @ref CORRADE_ENABLE_AVX_F16C, +@ref CORRADE_ENABLE_AVX_FMA or @ref CORRADE_ENABLE_AVX2 function attributes +instead of having to specify `-mavx` or `-mavx2` for the whole compilation +unit. This however can't reliably be done for `-mlzcnt`, `-mbmi`, `-mf16c` or +`-mfma` because then it could not be freely combined with other instruction +sets, only used alone. You have to enable these instructions globally in order +to use them on GCC 4.8. + +As AVX-512 is supported only since GCC 4.9, which doesn't need this workaround, +it's not handled here. +@see @relativeref{Corrade,Cpu}, @ref Cpu-usage-target-attributes, + @ref IntrinsicsSse2.h, @ref IntrinsicsSse3.h, @ref IntrinsicsSsse3.h, + @ref IntrinsicsSse4.h +*/ + +/* See https://gist.github.com/rygorous/f26f5f60284d9d9246f6 for more info. + If I wouldn't need the #define, it could be all put into a macro with + _Pragma to wrap around the include, but because I do, there has to be one + wrapper header for each include. */ + +/* So users don't need to include the SSE headers explicitly before, in correct + order */ +#include "Corrade/Utility/IntrinsicsSse4.h" + +/* Include just AVX instructions first. If we would add target("avx2") and + __AVX2__ here as well, it would cause the AVX instruction to be usable only + if target("avx2") is specified as well, which is not a good thing for + CORRADE_ENABLE_AVX, where the code should *only* use AVX at most. The AVX2 + instructions are included below by defining target("avx2") and directly + pulling in , and only doing that on GCC 4.8, as all other + compilers have everything already included from the top-level . + Same then goes for F16C, FMA, LZCNT and BMI1, which all also have such weird + interactions when pulled in together. + + I wonder what impact this has on optimization, but I don't care that much as + GCC 4.8 is mainly for backwards compatibility testing now, and not serious + performance use. In any case, one can always compile everything with + -march=native to get rid of such potential suboptimal optimization. */ +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("avx") +#pragma push_macro("__AVX__") +#ifndef __AVX__ +#define __AVX__ +#endif +#endif +#include +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma pop_macro("__AVX__") +#pragma GCC pop_options +#endif + +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("avx2") +#pragma push_macro("__AVX2__") +#ifndef __AVX2__ +#define __AVX2__ +#endif +#include +#pragma pop_macro("__AVX2__") +#pragma GCC pop_options +#endif diff --git a/src/Corrade/Utility/IntrinsicsSse2.h b/src/Corrade/Utility/IntrinsicsSse2.h new file mode 100644 index 000000000..e667cc71a --- /dev/null +++ b/src/Corrade/Utility/IntrinsicsSse2.h @@ -0,0 +1,57 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +/** @file +@brief Intrinsics for x86 SSE2 instructions +@m_since_latest + +Equivalent to @cpp #include @ce on most compilers except for GCC +4.8, where it contains an additional workaround to make the instructions +available with just the @ref CORRADE_ENABLE_SSE2 function attribute instead of +having to specify `-msse2` for the whole compilation unit. +@see @relativeref{Corrade,Cpu}, @ref Cpu-usage-target-attributes, + @ref IntrinsicsSse3.h, @ref IntrinsicsSsse3.h, @ref IntrinsicsSse4.h, + @ref IntrinsicsAvx.h +*/ + +/* See https://gist.github.com/rygorous/f26f5f60284d9d9246f6 for more info. + If it wouldn't be for the #define, it could be all put into a macro with + _Pragma to wrap around the include, but because I do, there has to be one + wrapper header for each include. */ + +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("sse2") +#pragma push_macro("__SSE2__") +#ifndef __SSE2__ +#define __SSE2__ +#endif +#endif +#include +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma pop_macro("__SSE2__") +#pragma GCC pop_options +#endif diff --git a/src/Corrade/Utility/IntrinsicsSse3.h b/src/Corrade/Utility/IntrinsicsSse3.h new file mode 100644 index 000000000..8e04be7ec --- /dev/null +++ b/src/Corrade/Utility/IntrinsicsSse3.h @@ -0,0 +1,61 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +/** @file +@brief Intrinsics for x86 SSE3 instructions +@m_since_latest + +Equivalent to @cpp #include @ce on most compilers except for GCC +4.8, where it contains an additional workaround to make the instructions +available with just the @ref CORRADE_ENABLE_SSE3 function attribute instead of +having to specify `-msse3` for the whole compilation unit. +@see @relativeref{Corrade,Cpu}, @ref Cpu-usage-target-attributes, + @ref IntrinsicsSse2.h, @ref IntrinsicsSsse3.h, @ref IntrinsicsSse4.h, + @ref IntrinsicsAvx.h +*/ + +/* See https://gist.github.com/rygorous/f26f5f60284d9d9246f6 for more info. + If I wouldn't need the #define, it could be all put into a macro with + _Pragma to wrap around the include, but because I do, there has to be one + wrapper header for each include. */ + +/* So users don't need to include the SSE2 header explicitly before, in correct + order */ +#include "Corrade/Utility/IntrinsicsSse2.h" + +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("sse3") +#pragma push_macro("__SSE3__") +#ifndef __SSE3__ +#define __SSE3__ +#endif +#endif +#include +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma pop_macro("__SSE3__") +#pragma GCC pop_options +#endif diff --git a/src/Corrade/Utility/IntrinsicsSse4.h b/src/Corrade/Utility/IntrinsicsSse4.h new file mode 100644 index 000000000..c6eae1c63 --- /dev/null +++ b/src/Corrade/Utility/IntrinsicsSse4.h @@ -0,0 +1,95 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +/** @file +@brief Intrinsics for x86 SSE4.1, SSE4.2 and POPCNT instructions +@m_since_latest + +Equivalent to @cpp #include @ce and +@cpp #include @ce on most compilers except for: + +- GCC 4.8, where it contains an additional workaround to make the + instructions available with just the @ref CORRADE_ENABLE_SSE41 and + @ref CORRADE_ENABLE_SSE42 function attributes instead of having to specify + `-msse4.1` or `-msse4.2` for the whole compilation unit. This however can't + reliably be done for `-mpopcnt` because then it could not be freely + combined with other instruction sets, only used alone. You have to enable + these instructions globally in order to use them on GCC 4.8. +- Clang < 7, where `__POPCNT__` has to be explicitly defined in order to + access the POPCNT instruction + +Because GCC puts both the SSE4.1 and the SSE4.2 instructions into the same +header and just guards each with a different macro, they have to be included +together, unlike with other SSE variants. +@see @relativeref{Corrade,Cpu}, @ref Cpu-usage-target-attributes, + @ref IntrinsicsSse2.h, @ref IntrinsicsSse3.h, @ref IntrinsicsSsse3.h, + @ref IntrinsicsAvx.h +*/ + +/* See https://gist.github.com/rygorous/f26f5f60284d9d9246f6 for more info. + If I wouldn't need the #define, it could be all put into a macro with + _Pragma to wrap around the include, but because I do, there has to be one + wrapper header for each include. */ + +/* So users don't need to include the SSE2, SSE3 and SSSE3 headers explicitly + before, in correct order */ +#include "Corrade/Utility/IntrinsicsSsse3.h" + +/* Unfortunately, as opposed to AVX, the SSE4.1 and 4.2 intrinsics live in the + exact same header, which for GCC 4.8 means we have to subsequently have + target("sse4.2") again in order to call even SSE4.1. This is the reason why + CORRADE_ENABLE_SSE41 is not defined there. */ +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("sse4.1") +#pragma GCC target("sse4.2") +#pragma push_macro("__SSE4_1__") +#pragma push_macro("__SSE4_2__") +#ifndef __SSE4_1__ +#define __SSE4_1__ +#endif +#ifndef __SSE4_2__ +#define __SSE4_2__ +#endif +#endif +/* https://github.com/llvm/llvm-project/commit/092d42557b6c70d32b0b9362e4a8db9566369ddc + says this was an accidental omission, so this should do no harm */ +#if defined(CORRADE_TARGET_CLANG) && __clang_major__ < 7 +#pragma push_macro("__POPCNT__") +#ifndef __POPCNT__ +#define __POPCNT__ +#endif +#endif +#include +#include +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma pop_macro("__SSE4_1__") +#pragma pop_macro("__SSE4_2__") +#pragma GCC pop_options +#endif +#if defined(CORRADE_TARGET_CLANG) && __clang_major__ < 7 +#pragma pop_macro("__POPCNT__") +#endif diff --git a/src/Corrade/Utility/IntrinsicsSsse3.h b/src/Corrade/Utility/IntrinsicsSsse3.h new file mode 100644 index 000000000..c87fc7e9f --- /dev/null +++ b/src/Corrade/Utility/IntrinsicsSsse3.h @@ -0,0 +1,61 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +/** @file +@brief Intrinsics for x86 SSSE3 instructions +@m_since_latest + +Equivalent to @cpp #include @ce on most compilers except for GCC +4.8, where it contains an additional workaround to make the instructions +available with just the @ref CORRADE_ENABLE_SSSE3 function attribute instead of +having to specify `-mssse3` for the whole compilation unit. +@see @relativeref{Corrade,Cpu}, @ref Cpu-usage-target-attributes, + @ref IntrinsicsSse2.h, @ref IntrinsicsSse3.h, @ref IntrinsicsSse4.h, + @ref IntrinsicsAvx.h +*/ + +/* See https://gist.github.com/rygorous/f26f5f60284d9d9246f6 for more info. + If it wouldn't be for the #define, it could be all put into a macro with + _Pragma to wrap around the include, but because I do, there has to be one + wrapper header for each include. */ + +/* So users don't need to include the SSE2 and SSE3 headers explicitly before, + in correct order */ +#include "Corrade/Utility/IntrinsicsSse3.h" + +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("ssse3") +#pragma push_macro("__SSSE3__") +#ifndef __SSSE3__ +#define __SSSE3__ +#endif +#endif +#include +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma pop_macro("__SSSE3__") +#pragma GCC pop_options +#endif diff --git a/src/Corrade/Utility/Memory.h b/src/Corrade/Utility/Memory.h index 6dd1d5a09..6a3712398 100644 --- a/src/Corrade/Utility/Memory.h +++ b/src/Corrade/Utility/Memory.h @@ -105,6 +105,8 @@ loop or a call to @ref std::memset(). constructor on all elements manually using placement new, @ref std::uninitialized_copy() or similar --- see the function docs for an example. + +@see @ref Cpu */ template inline Containers::Array allocateAligned(std::size_t size); @@ -121,7 +123,7 @@ differing behavior for trivial types it's better to explicitly use either the Implemented via @ref allocateAligned(NoInitT, std::size_t) with a loop calling the constructors on the returned allocation in case of non-trivial types. -@see @ref allocateAligned(ValueInitT, std::size_t) +@see @ref allocateAligned(ValueInitT, std::size_t), @ref Cpu */ template Containers::Array allocateAligned(DefaultInitT, std::size_t size); @@ -133,7 +135,7 @@ Same as @ref allocateAligned(std::size_t), just more explicit. Implemented via @ref allocateAligned(NoInitT, std::size_t) with either a @ref std::memset() or a loop calling the constructors on the returned allocation. -@see @ref allocateAligned(DefaultInitT, std::size_t) +@see @ref allocateAligned(DefaultInitT, std::size_t), @ref Cpu */ template Containers::Array allocateAligned(ValueInitT, std::size_t size); @@ -153,7 +155,7 @@ uninitialized memory: @snippet Utility.cpp allocateAligned-NoInit @see @ref allocateAligned(DefaultInitT, std::size_t), - @ref allocateAligned(ValueInitT, std::size_t) + @ref allocateAligned(ValueInitT, std::size_t), @ref Cpu */ template Containers::Array allocateAligned(NoInitT, std::size_t size); diff --git a/src/Corrade/Utility/Test/CMakeLists.txt b/src/Corrade/Utility/Test/CMakeLists.txt index 0ba8c3739..a0669b539 100644 --- a/src/Corrade/Utility/Test/CMakeLists.txt +++ b/src/Corrade/Utility/Test/CMakeLists.txt @@ -39,10 +39,16 @@ endif() configure_file(${CMAKE_CURRENT_SOURCE_DIR}/configure.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/configure.h) -corrade_add_test(UtilityAlgorithmsTest AlgorithmsTest.cpp LIBRARIES CorradeUtilityTestLib) +# In all following corrade_add_test() macros, if a test wants to use +# CorradeUtilityTestLib, it has to link to CorradeTestSuiteTestLib instead. +# Otherwise the implicitly linked CorradeTestSuite would drag in CorradeUtility +# in addition to CorradeUtilityTestLib, leading to ODR violations and making +# ASan builds fail. + +corrade_add_test(UtilityAlgorithmsTest AlgorithmsTest.cpp LIBRARIES CorradeTestSuiteTestLib) target_compile_definitions(UtilityAlgorithmsTest PRIVATE "CORRADE_GRACEFUL_ASSERT") -corrade_add_test(UtilityArgumentsTest ArgumentsTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(UtilityArgumentsTest ArgumentsTest.cpp LIBRARIES CorradeTestSuiteTestLib) set_tests_properties(UtilityArgumentsTest PROPERTIES ENVIRONMENT "ARGUMENTSTEST_SIZE=1337;ARGUMENTSTEST_VERBOSE=ON;ARGUMENTSTEST_COLOR=OFF;ARGUMENTSTEST_UNICODE=hýždě") @@ -53,10 +59,10 @@ target_include_directories(DebugAssertTestObjects PRIVATE $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp) + ${PROJECT_SOURCE_DIR}/src/dummy.cpp) # XCode workaround, see file comment for details corrade_add_test(UtilityDebugAssertTest $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp) + ${PROJECT_SOURCE_DIR}/src/dummy.cpp) # XCode workaround, see file comment for details # WILL_FAIL doesn't work for abort() on desktop, test this only on embedded # then. Oh well. Also the tests could be just one executable added multiple # times with different arguments, but corrade_add_test() doesn't support that, @@ -65,55 +71,55 @@ if(CORRADE_TARGET_EMSCRIPTEN OR CORRADE_TARGET_ANDROID) foreach(debug "" "Debug") corrade_add_test(Utility${debug}AssertTestFailAssert $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-assert true) set_tests_properties(Utility${debug}AssertTestFailAssert PROPERTIES PASS_REGULAR_EXPRESSION "A should be zero") corrade_add_test(Utility${debug}AssertTestFailConstexprAssert $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-constexpr-assert true) set_tests_properties(Utility${debug}AssertTestFailConstexprAssert PROPERTIES PASS_REGULAR_EXPRESSION "b can't be zero") corrade_add_test(Utility${debug}AssertTestFailInternalAssert $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-internal-assert true) set_tests_properties(Utility${debug}AssertTestFailInternalAssert PROPERTIES PASS_REGULAR_EXPRESSION "Assertion b && !_failInternalAssert failed at ") corrade_add_test(Utility${debug}AssertTestFailInternalC___Assert $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-internal-constexpr-assert true) set_tests_properties(Utility${debug}AssertTestFailInternalC___Assert PROPERTIES PASS_REGULAR_EXPRESSION "Assertion b failed at ") corrade_add_test(Utility${debug}AssertTestFailAssertOutput $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-assert-output true) set_tests_properties(Utility${debug}AssertTestFailAssertOutput PROPERTIES PASS_REGULAR_EXPRESSION "foo\\(\\) should succeed") corrade_add_test(Utility${debug}AssertTestFailInternalA___Output $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-internal-assert-output true) set_tests_properties(Utility${debug}AssertTestFailInternalA___Output PROPERTIES PASS_REGULAR_EXPRESSION "Assertion foo\\(\\) && !_failInternalAssertOutput failed at ") corrade_add_test(Utility${debug}AssertTestFailInter___Expression $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-internal-assert-expression true) set_tests_properties(Utility${debug}AssertTestFailInter___Expression PROPERTIES PASS_REGULAR_EXPRESSION "Assertion c \\+ \\(_failInternalAssertExpression \\? -3 : 3\\) failed at ") corrade_add_test(Utility${debug}AssertTestFailAssertUnreachable $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-assert-unreachable true) set_tests_properties(Utility${debug}AssertTestFailAssertUnreachable PROPERTIES PASS_REGULAR_EXPRESSION "C should be 3") corrade_add_test(Utility${debug}AssertTestFailInte___Unreachable $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-internal-assert-unreachable true) set_tests_properties(Utility${debug}AssertTestFailInte___Unreachable PROPERTIES PASS_REGULAR_EXPRESSION "Reached unreachable code at ") @@ -149,7 +155,7 @@ corrade_add_test(UtilityEndiannessTest EndiannessTest.cpp) corrade_add_test(UtilityErrorStringTest ErrorStringTest.cpp) corrade_add_test(UtilityMurmurHash2Test MurmurHash2Test.cpp) corrade_add_test(UtilityConfigurationTest ConfigurationTest.cpp - LIBRARIES CorradeUtilityTestLib + LIBRARIES CorradeTestSuiteTestLib FILES ConfigurationTestFiles/bom.conf ConfigurationTestFiles/comments.conf @@ -224,14 +230,14 @@ corrade_add_test(UtilityFatalTest FatalTest.cpp) set_tests_properties(UtilityFatalTest PROPERTIES WILL_FAIL ON) corrade_add_test(UtilityJsonTest JsonTest.cpp - LIBRARIES CorradeUtilityTestLib + LIBRARIES CorradeTestSuiteTestLib FILES JsonTestFiles/error.json JsonTestFiles/parse-error.json) target_compile_definitions(UtilityJsonTest PRIVATE "CORRADE_GRACEFUL_ASSERT") target_include_directories(UtilityJsonTest PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) -corrade_add_test(UtilityJsonWriterTest JsonWriterTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(UtilityJsonWriterTest JsonWriterTest.cpp LIBRARIES CorradeTestSuiteTestLib) target_include_directories(UtilityJsonWriterTest PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) corrade_add_test(UtilityMemoryTest MemoryTest.cpp) @@ -269,7 +275,7 @@ if(CORRADE_TARGET_UNIX) endif() target_include_directories(UtilityPathTest PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) -corrade_add_test(UtilityFormatTest FormatTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(UtilityFormatTest FormatTest.cpp LIBRARIES CorradeTestSuiteTestLib) target_include_directories(UtilityFormatTest PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) corrade_add_test(UtilityHashDigestTest HashDigestTest.cpp) @@ -290,12 +296,12 @@ if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT CMAKE_CXX_COMPILER_VERSION VERS CORRADE_CXX_STANDARD 17) endif() -corrade_add_test(UtilityStringTest StringTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(UtilityStringTest StringTest.cpp LIBRARIES CorradeTestSuiteTestLib) corrade_add_test(UtilityStringBenchmark StringBenchmark.cpp) corrade_add_test(UtilitySystemTest SystemTest.cpp) corrade_add_test(UtilityTweakableParserTest TweakableParserTest.cpp) corrade_add_test(UtilityTypeTraitsTest TypeTraitsTest.cpp) -corrade_add_test(UtilityUnicodeTest UnicodeTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(UtilityUnicodeTest UnicodeTest.cpp LIBRARIES CorradeTestSuiteTestLib) # Compiled-in resource test corrade_add_resource(ResourceTestData ResourceTestFiles/resources.conf) @@ -306,7 +312,7 @@ corrade_add_test(UtilityResourceTest ${ResourceTestData} ${ResourceTestEmptyFileData} ${ResourceTestNothingData} - LIBRARIES CorradeUtilityTestLib + LIBRARIES CorradeTestSuiteTestLib FILES ResourceTestFiles/consequence.bin # Referenced from resources-overriden.conf @@ -329,7 +335,7 @@ add_library(ResourceTestDataLib STATIC ${ResourceTestData} target_compile_definitions(ResourceTestDataLib PRIVATE "CORRADE_AUTOMATIC_INITIALIZER=CORRADE_NOOP" "CORRADE_AUTOMATIC_FINALIZER=CORRADE_NOOP") -target_link_libraries(ResourceTestDataLib CorradeUtility) +target_link_libraries(ResourceTestDataLib PUBLIC CorradeUtility) corrade_add_test(UtilityResourceStaticTest ResourceStaticTest.cpp LIBRARIES ResourceTestDataLib FILES diff --git a/src/Corrade/Utility/Test/cpuVariantHelpers.h b/src/Corrade/Utility/Test/cpuVariantHelpers.h new file mode 100644 index 000000000..271dce35d --- /dev/null +++ b/src/Corrade/Utility/Test/cpuVariantHelpers.h @@ -0,0 +1,73 @@ +#ifndef Corrade_Utility_Test_cpuVariantHelpers_h +#define Corrade_Utility_Test_cpuVariantHelpers_h +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include + +#include "Corrade/Cpu.h" +#include "Corrade/Containers/String.h" +#include "Corrade/Containers/StringStl.h" +#include "Corrade/Utility/Assert.h" + +namespace Corrade { namespace Utility { namespace Test { + +template constexpr std::size_t cpuVariantCount(T(&)[size]) { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + return size; + #else + return 1; + #endif +} + +template inline Containers::String cpuVariantName(T& data) { + std::ostringstream out; + Utility::Debug{&out, Utility::Debug::Flag::NoNewlineAtTheEnd} << Utility::Debug::packed << data.features; + return out.str(); +} + +template inline const T& cpuVariantCompiled(const T(&data)[size]) { + const Cpu::Features features = + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + Cpu::runtimeFeatures() + #else + Cpu::compiledFeatures() + #endif + ; + for(std::size_t i = size; i != 0; --i) + if(features >= data[i - 1].features) + return data[i - 1]; + + CORRADE_INTERNAL_ASSERT_UNREACHABLE(); +} + +template inline bool isCpuVariantSupported(T& data) { + return Cpu::runtimeFeatures() >= data.features; +} + +}}} + +#endif diff --git a/src/Corrade/Utility/visibility.h b/src/Corrade/Utility/visibility.h index dbdf160da..ab75a1d0e 100644 --- a/src/Corrade/Utility/visibility.h +++ b/src/Corrade/Utility/visibility.h @@ -45,4 +45,37 @@ #define CORRADE_UTILITY_LOCAL #endif +/* Function-pointer-based CPU dispatch, exposing also the runtime dispatcher + implementation */ +#if defined(CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH) || (defined(CORRADE_BUILD_CPU_RUNTIME_DISPATCH) && !defined(CORRADE_CPU_USE_IFUNC)) + #define CORRADE_UTILITY_CPU_DISPATCHER_DECLARATION(name) \ + CORRADE_UTILITY_EXPORT decltype(name) name ## Implementation(Cpu::Features); + #define CORRADE_UTILITY_CPU_DISPATCHER(...) CORRADE_CPU_DISPATCHER(__VA_ARGS__) + #define CORRADE_UTILITY_CPU_DISPATCHED_DECLARATION(name) (*name) + #define CORRADE_UTILITY_CPU_DISPATCHED(dispatcher, ...) \ + CORRADE_CPU_DISPATCHED_POINTER(dispatcher, __VA_ARGS__) CORRADE_NOOP + #define CORRADE_UTILITY_CPU_MAYBE_UNUSED + +/* IFUNC or compile-time CPU dispatch, runtime dispatcher is not exposed */ +#else + #define CORRADE_UTILITY_CPU_DISPATCHER_DECLARATION(name) + #define CORRADE_UTILITY_CPU_DISPATCHED_DECLARATION(name) (name) + /* Runtime dispatcher implementation is either hidden or not present at + all */ + #if defined(CORRADE_BUILD_CPU_RUNTIME_DISPATCH) && defined(CORRADE_CPU_USE_IFUNC) + #define CORRADE_UTILITY_CPU_DISPATCHER(...) \ + namespace { CORRADE_CPU_DISPATCHER(__VA_ARGS__) } + #define CORRADE_UTILITY_CPU_DISPATCHED(dispatcher, ...) \ + CORRADE_CPU_DISPATCHED_IFUNC(dispatcher, __VA_ARGS__) CORRADE_NOOP + #define CORRADE_UTILITY_CPU_MAYBE_UNUSED + #elif !defined(CORRADE_BUILD_CPU_RUNTIME_DISPATCH) + #define CORRADE_UTILITY_CPU_DISPATCHER(...) + #define CORRADE_UTILITY_CPU_DISPATCHED(dispatcher, ...) \ + __VA_ARGS__ CORRADE_PASSTHROUGH + #define CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_UNUSED + #else + #error mosra messed up! + #endif +#endif + #endif diff --git a/src/Corrade/configure.h.cmake b/src/Corrade/configure.h.cmake index 8d7b294d6..a6e00d0d3 100644 --- a/src/Corrade/configure.h.cmake +++ b/src/Corrade/configure.h.cmake @@ -34,6 +34,7 @@ #cmakedefine CORRADE_BUILD_STATIC #cmakedefine CORRADE_BUILD_STATIC_UNIQUE_GLOBALS #cmakedefine CORRADE_BUILD_MULTITHREADED +#cmakedefine CORRADE_BUILD_CPU_RUNTIME_DISPATCH #cmakedefine CORRADE_TARGET_APPLE #cmakedefine CORRADE_TARGET_IOS @@ -44,6 +45,7 @@ #cmakedefine CORRADE_TARGET_EMSCRIPTEN #cmakedefine CORRADE_TARGET_ANDROID +#cmakedefine CORRADE_CPU_USE_IFUNC #cmakedefine CORRADE_PLUGINMANAGER_NO_DYNAMIC_PLUGIN_SUPPORT #cmakedefine CORRADE_TESTSUITE_TARGET_XCTEST #cmakedefine CORRADE_UTILITY_USE_ANSI_COLORS @@ -181,10 +183,11 @@ #define CORRADE_BIG_ENDIAN #endif -/* Compile-time SIMD detection */ +/* Compile-time CPU feature detection */ #ifdef CORRADE_TARGET_X86 /* SSE on GCC: https://stackoverflow.com/a/28939692 */ +/** @todo check also for Clang, maybe clang-cl defines these too? */ #ifdef CORRADE_TARGET_GCC #ifdef __SSE2__ #define CORRADE_TARGET_SSE2 @@ -232,9 +235,56 @@ #endif #endif -/* On GCC, F16C and FMA have its own define, on MSVC it's implied by /arch:AVX2 - (source: https://docs.microsoft.com/en-us/cpp/build/reference/arch-x86 ... - or at least the FMA instructions, no word about F16C). */ +/* POPCNT, LZCNT and BMI1 on GCC, queried wth + `gcc -mpopcnt -dM -E - | grep POPCNT`, and equivalent for others. */ +#ifdef CORRADE_TARGET_GCC +#ifdef __POPCNT__ +#define CORRADE_TARGET_POPCNT +#endif +#ifdef __LZCNT__ +#define CORRADE_TARGET_LZCNT +#endif +#ifdef __BMI__ +#define CORRADE_TARGET_BMI1 +#endif + +/* There doesn't seem to be any equivalent on MSVC, + https://github.com/kimwalisch/libpopcnt assumes POPCNT is on x86 MSVC + always, and LZCNT has encoding compatible with BSR so if not available it'll + not crash but produce wrong results, sometimes. Enabling them always feels a + bit too much, so instead going with what clang-cl uses. There /arch:AVX + matches -march=sandybridge: + https://github.com/llvm/llvm-project/blob/6542cb55a3eb115b1c3592514590a19987ffc498/clang/lib/Driver/ToolChains/Arch/X86.cpp#L46-L58 + and `echo | clang -march=sandybridge -dM -E -` lists only POPCNT, while + /arch:AVX2 matches -march=haswell, which lists also LZCNT, BMI and BMI2. */ +#elif defined(CORRADE_TARGET_MSVC) +/* For extra robustness on clang-cl check the macros explicitly -- as with + other AVX+ intrinsics, these are only included if the corresponding macro is + defined as well. Failing to do so would mean the + CORRADE_ENABLE_AVX_{POPCNT,LZCNT,BMI1} macros are defined always, + incorrectly implying presence of these intrinsics. */ +#ifdef __AVX__ +#if !defined(CORRADE_TARGET_CLANG_CL) || defined(__POPCNT__) +#define CORRADE_TARGET_POPCNT +#endif +#endif +#ifdef __AVX2__ +#if !defined(CORRADE_TARGET_CLANG_CL) || defined(__LZCNT__) +#define CORRADE_TARGET_LZCNT +#endif +#if !defined(CORRADE_TARGET_CLANG_CL) || defined(__BMI__) +#define CORRADE_TARGET_BMI1 +#endif +#endif +#endif + +/* On GCC, F16C and FMA have its own define, on MSVC FMA is implied by + /arch:AVX2 (https://docs.microsoft.com/en-us/cpp/build/reference/arch-x86), + no mention of F16C but https://walbourn.github.io/directxmath-f16c-and-fma/ + says it's like that so I'll believe that. However, comments below + https://stackoverflow.com/a/50829580 say there is a Via processor with AVX2 + but no FMA, so then the __AVX2__ check isn't really bulletproof. Use runtime + detection where possible, please. */ #ifdef CORRADE_TARGET_GCC #ifdef __F16C__ #define CORRADE_TARGET_AVX_F16C @@ -243,9 +293,20 @@ #define CORRADE_TARGET_AVX_FMA #endif #elif defined(CORRADE_TARGET_MSVC) && defined(__AVX2__) +/* On clang-cl /arch:AVX2 matches -march=haswell: + https://github.com/llvm/llvm-project/blob/6542cb55a3eb115b1c3592514590a19987ffc498/clang/lib/Driver/ToolChains/Arch/X86.cpp#L46-L58 + And `echo | clang -march=haswell -dM -E -` lists both F16C and FMA. However, + for robustness, check the macros explicitly -- as with other AVX+ intrinsics + on clang-cl, these are only included if the corresponding macro is defined + as well. Failing to do so would mean the CORRADE_ENABLE_AVX_{16C,FMA} macros + are defined always, incorrectly implying presence of these intrinsics. */ +#if !defined(CORRADE_TARGET_CLANG_CL) || defined(__F16C__) #define CORRADE_TARGET_AVX_F16C +#endif +#if !defined(CORRADE_TARGET_CLANG_CL) || defined(__FMA__) #define CORRADE_TARGET_AVX_FMA #endif +#endif /* https://stackoverflow.com/a/37056771, confirmed on Android NDK Clang that __ARM_NEON is indeed still set. For MSVC, according to @@ -256,24 +317,34 @@ #elif defined(CORRADE_TARGET_ARM) #ifdef __ARM_NEON #define CORRADE_TARGET_NEON -/* Conservatively mark half-floats as supported only if the IEEE variant is - supported and not the ARM-specific variant that trades one extra exponent - value for a lack of inf and NaN support (ARM C Language Extensions 1.1, - §6.5.2: https://developer.arm.com/documentation/ihi0053/b/) */ -#if __ARM_FP16_FORMAT_IEEE && (__ARM_NEON_FP & 0x02) -#define CORRADE_TARGET_NEON_FP16 -#endif /* NEON FMA is available only if __ARM_FEATURE_FMA is defined and some bits of __ARM_NEON_FP as well (ARM C Language Extensions 1.1, §6.5.5: - https://developer.arm.com/documentation/ihi0053/b/) */ -#if defined(__ARM_FEATURE_FMA) && __ARM_NEON_FP + https://developer.arm.com/documentation/ihi0053/b/). On AAArch64 NEON is + implicitly supported and __ARM_NEON_FP might not be defined (Android Clang + defines it but GCC 9 on Ubuntu ARM64 not), so check for __aarch64__ as + well. */ +#if defined(__ARM_FEATURE_FMA) && (__ARM_NEON_FP || defined(__aarch64__)) #define CORRADE_TARGET_NEON_FMA #endif +/* There's no linkable documentation for anything and the PDF is stupid. But, + given the FP16 instructions implemented in GCC and Clang are guarded by this + macro, it should be alright: + https://gcc.gnu.org/legacy-ml/gcc-patches/2016-06/msg00460.html */ +#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +#define CORRADE_TARGET_NEON_FP16 +#endif #endif -/* Undocumented, checked via `echo | em++ -x c++ -dM -E - -msimd128` */ +/* Undocumented, checked via `echo | em++ -x c++ -dM -E - -msimd128`. + Restricting to the finalized SIMD variant, which is since Clang 13: + https://github.com/llvm/llvm-project/commit/502f54049d17f5a107f833596fb2c31297a99773 + Emscripten 2.0.13 sets Clang 13 as the minimum, however it doesn't imply + that emsdk 2.0.13 actually contains the final Clang 13. That's only since + 2.0.18, thus to avoid nasty issues we have to check Emscripten version as + well :( + https://github.com/emscripten-core/emscripten/commit/deab7783df407b260f46352ffad2a77ca8fb0a4c */ #elif defined(CORRADE_TARGET_WASM) -#ifdef __wasm_simd128__ +#if defined(__wasm_simd128__) && __clang_major__ >= 13 && __EMSCRIPTEN_major__*10000 + __EMSCRIPTEN_minor__*100 + __EMSCRIPTEN_tiny__ >= 20018 #define CORRADE_TARGET_SIMD128 #endif #endif @@ -313,4 +384,23 @@ static_assert(sizeof(1 ? "" : "") == 1, "enabled or ensure /permissive- is set for all files that include Corrade " "headers."); #endif +/* Kill switch for when presence of a sanitizer is detected and + CORRADE_CPU_USE_IFUNC is enabled. Unfortunately in our case the + __attribute__((no_sanitize_address)) workaround as described on + https://github.com/google/sanitizers/issues/342 doesn't work / can't be used + because it would mean marking basically everything including the actual + implementation that's being dispatched to. */ +#ifdef CORRADE_CPU_USE_IFUNC +#ifdef __has_feature +#if __has_feature(address_sanitizer) || __has_feature(thread_sanitizer) || __has_feature(memory_sanitizer) || __has_feature(undefined_behavior_sanitizer) +#define _CORRADE_SANITIZER_IFUNC_DETECTED +#endif +#elif defined(__SANITIZE_ADDRESS__) || defined(__SANITIZE_THREAD__) +#define _CORRADE_SANITIZER_IFUNC_DETECTED +#endif +#ifdef _CORRADE_SANITIZER_IFUNC_DETECTED +#error Corrade was built with CORRADE_CPU_USE_IFUNC, which is incompatible with sanitizers. Rebuild without this option or disable sanitizers. +#endif +#endif + #endif // kate: hl c++